claude-bot
595c382ac7
Закрывает F-1-пробел ORCH-068: терминал-исключение и in-memory dedup (изначально только F-2) распространены на gate-side путь реконсилятора, устраняя ложное «🔧 reconciler: ET-002 done разблокирована (потерян webhook)» (особенно после рестарта). - D1: новый _resolve_issue_status — один сетевой резолв Plane-статуса задачи за тик (states, groups, state_uuid) после дешёвых локальных гардов; never-raise -> ({}, {}, None) при сбое. - D2: безусловный терминал-скип ДО Guard 2 (группа Plane completed/ cancelled, fallback на логические ключи done/cancelled, либо стадия в БД орка ∈ {done, cancelled}); skipped_terminal_total++, не подчинён reconcile_skip_blocked_enabled. - D3: _is_blocked_or_needs_input переиспользует резолв D1 (опц. аргументы, _UNSET -> самостоятельный резолв для прямых/легаси-вызовов; 1:1). - D4: вызов _note_unblock на F-1 теперь передаёт state_uuid -> dedup работает на обоих путях (deduped_total++ на повторе). Анти-регресс: легитимный unblock не-терминальной застрявшей задачи по-прежнему advance + один Telegram. STAGE_TRANSITIONS / QG_CHECKS / схема БД / сигнатуры advance_*/_note_unblock / форма status() / новые флаги — без изменений; never-raise сохранён. Тесты: tests/test_reconciler.py TC-86-01..09/11, tests/test_reconciler_plane.py TC-86-10. Полный прогон зелёный (1069). Refs: ORCH-086 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
977 lines
39 KiB
Python
977 lines
39 KiB
Python
"""ORCH-053: tests for the gate-side stuck-task reconciler (F-1) + lifecycle.
|
||
|
||
These cover the F-1 sweeper (``Reconciler.reconcile_gate_once``), the per-stage
|
||
grace / config (``grace_for_stage``), the no-spam guarantee, the analysis carve-
|
||
out (AC-16), never-raise isolation, the kill-switch, the unblock observability
|
||
(AC-12 / F-4) and the restart-safe daemon thread (AC-11).
|
||
|
||
Everything that touches the network (the quality gate, Plane sync, Telegram) is
|
||
mocked at the src.stage_engine / src.reconciler level so the reconciler runs
|
||
against a real isolated sqlite DB (same convention as test_stage_engine.py).
|
||
"""
|
||
|
||
import os
|
||
import tempfile
|
||
|
||
import pytest
|
||
|
||
# Isolated test DB (set BEFORE importing src.* so settings picks it up).
|
||
_test_db = os.path.join(tempfile.gettempdir(), "test_orchestrator_reconciler.db")
|
||
os.environ["ORCH_DB_PATH"] = _test_db
|
||
os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
|
||
os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
|
||
os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
|
||
|
||
from unittest.mock import MagicMock # noqa: E402
|
||
|
||
import src.db as _db # noqa: E402
|
||
from src.db import init_db, get_db, enqueue_job # noqa: E402
|
||
from src import stage_engine # noqa: E402
|
||
from src import reconciler as reconciler_mod # noqa: E402
|
||
from src.reconciler import Reconciler, grace_for_stage # noqa: E402
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixtures
|
||
# ---------------------------------------------------------------------------
|
||
@pytest.fixture(autouse=True)
|
||
def fresh_db(monkeypatch):
|
||
"""Fresh isolated DB per test."""
|
||
monkeypatch.setattr(_db.settings, "db_path", _test_db)
|
||
if os.path.exists(_test_db):
|
||
os.unlink(_test_db)
|
||
init_db()
|
||
yield
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def silence_side_effects(monkeypatch):
|
||
"""No-op every Plane/Telegram/notification side effect in the engine so the
|
||
real advance_stage runs deterministically and offline."""
|
||
for name in (
|
||
"notify_stage_change",
|
||
"notify_qg_failure",
|
||
"notify_approve_requested",
|
||
"notify_error",
|
||
"send_telegram",
|
||
"plane_notify_stage",
|
||
"plane_notify_qg",
|
||
"plane_add_comment",
|
||
"set_issue_in_review",
|
||
"set_issue_needs_input",
|
||
"set_issue_in_progress",
|
||
"set_issue_blocked",
|
||
"set_issue_done",
|
||
):
|
||
monkeypatch.setattr(stage_engine, name, MagicMock(), raising=False)
|
||
|
||
|
||
def _make_task(stage, *, repo="enduro-trails", branch="feature/ET-001-x",
|
||
wi="ET-001", age_s=None):
|
||
"""Insert a task; if age_s is given, backdate updated_at by that many secs."""
|
||
conn = get_db()
|
||
cur = conn.execute(
|
||
"INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
|
||
"VALUES (?, ?, ?, ?, ?)",
|
||
(f"plane-{wi}", wi, repo, branch, stage),
|
||
)
|
||
task_id = cur.lastrowid
|
||
if age_s is not None:
|
||
conn.execute(
|
||
"UPDATE tasks SET updated_at = datetime('now', ?) WHERE id = ?",
|
||
(f"-{int(age_s)} seconds", task_id),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
return task_id
|
||
|
||
|
||
def _stage_of(task_id):
|
||
conn = get_db()
|
||
row = conn.execute("SELECT stage FROM tasks WHERE id = ?", (task_id,)).fetchone()
|
||
conn.close()
|
||
return row["stage"]
|
||
|
||
|
||
def _jobs_for(task_id, agent=None):
|
||
conn = get_db()
|
||
if agent:
|
||
rows = conn.execute(
|
||
"SELECT * FROM jobs WHERE task_id = ? AND agent = ?", (task_id, agent)
|
||
).fetchall()
|
||
else:
|
||
rows = conn.execute(
|
||
"SELECT * FROM jobs WHERE task_id = ?", (task_id,)
|
||
).fetchall()
|
||
conn.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def _green_ci(monkeypatch, value=(True, "CI green")):
|
||
"""Patch the check_ci_green entry in QG_CHECKS; return the mock."""
|
||
m = MagicMock(return_value=value)
|
||
monkeypatch.setitem(stage_engine.QG_CHECKS, "check_ci_green", m)
|
||
return m
|
||
|
||
|
||
# --- ORCH-060 fixtures / helpers -------------------------------------------
|
||
# State uuids the default "not blocked" fixture maps Blocked / Needs Input to.
|
||
_BLOCKED_UUID = "blocked-state-uuid"
|
||
_NEEDS_INPUT_UUID = "needs-input-state-uuid"
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def plane_state_not_blocked(monkeypatch):
|
||
"""ORCH-060 Guard 2 boundary: by default Plane says the issue is NOT in a
|
||
human gate, so the F-1 happy path runs deterministically offline (no real
|
||
httpx call). Tests that exercise Guard 2 override ``fetch_issue_state`` to
|
||
return ``_BLOCKED_UUID`` / ``_NEEDS_INPUT_UUID`` (or raise)."""
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value="some-non-gated-state"),
|
||
)
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "get_project_states",
|
||
MagicMock(return_value={
|
||
"blocked": _BLOCKED_UUID,
|
||
"needs_input": _NEEDS_INPUT_UUID,
|
||
}),
|
||
)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.projects, "get_project_by_repo",
|
||
MagicMock(return_value=MagicMock(plane_project_id="proj-test")),
|
||
)
|
||
|
||
|
||
def _add_dev_runs(task_id, n, agent="developer"):
|
||
"""Model N developer retries by inserting N agent_runs rows (ORCH-060)."""
|
||
conn = get_db()
|
||
for _ in range(n):
|
||
conn.execute(
|
||
"INSERT INTO agent_runs (task_id, agent) VALUES (?, ?)",
|
||
(task_id, agent),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-01: happy path — stuck development task is advanced to review
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc01_advances_stuck_development_task(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600) # well past grace
|
||
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "review"
|
||
reviewer_jobs = _jobs_for(task_id, "reviewer")
|
||
assert len(reviewer_jobs) == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-02: source of truth is the gate — advance goes through advance_stage
|
||
# with finished_agent=None (no own update_task_stage/enqueue_job).
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc02_advances_via_advance_stage_finished_agent_none(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
# advance_if_gate_passed resolves advance_stage as a module global.
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
|
||
task_id = _make_task("development", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert spy.call_count == 1
|
||
# finished_agent must be None (the webhook path).
|
||
_args, kwargs = spy.call_args
|
||
assert kwargs.get("finished_agent", "MISSING") is None
|
||
assert spy.call_args.args[0] == task_id
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-03: task with an active job is skipped — gate not evaluated, no advance.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc03_active_job_skipped(monkeypatch):
|
||
ci = _green_ci(monkeypatch)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
|
||
task_id = _make_task("development", age_s=3600)
|
||
enqueue_job("reviewer", "enduro-trails", task_id=task_id) # active (queued)
|
||
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
ci.assert_not_called()
|
||
spy.assert_not_called()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-04: per-stage grace — fresh task untouched, at-threshold task eligible.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc04_grace_boundary(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600)
|
||
_green_ci(monkeypatch)
|
||
|
||
fresh = _make_task("development", branch="feature/ET-002-fresh",
|
||
wi="ET-002", age_s=10) # < grace -> untouched
|
||
stuck = _make_task("development", branch="feature/ET-003-stuck",
|
||
wi="ET-003", age_s=3600) # >= grace -> advanced
|
||
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(fresh) == "development"
|
||
assert _stage_of(stuck) == "review"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-05: grace_for_stage reads overrides JSON; bad JSON -> default, no crash.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc05_grace_for_stage_overrides(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.settings,
|
||
"reconcile_grace_overrides_json",
|
||
'{"development": 30, "review": 7200}',
|
||
)
|
||
assert grace_for_stage("development") == 30
|
||
assert grace_for_stage("review") == 7200
|
||
# missing key -> default
|
||
assert grace_for_stage("testing") == 600
|
||
|
||
|
||
def test_tc05_grace_for_stage_invalid_json_falls_back(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.settings, "reconcile_grace_overrides_json", "{not valid json"
|
||
)
|
||
# Must not raise, must fall back to the default.
|
||
assert grace_for_stage("development") == 600
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-06: no spam — a stable-red gate never advances and never notifies, even
|
||
# across many ticks.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc06_red_gate_no_spam(monkeypatch):
|
||
_green_ci(monkeypatch, value=(False, "CI red"))
|
||
task_id = _make_task("development", age_s=3600)
|
||
|
||
rec = Reconciler()
|
||
for _ in range(5):
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
# The QG-failure notification branch inside advance_stage must never fire,
|
||
# because advance_if_gate_passed returns None on a red gate (no advance call).
|
||
stage_engine.notify_qg_failure.assert_not_called()
|
||
stage_engine.plane_notify_qg.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-07: silence when in sync — done / busy / within-grace tasks => no advance.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc07_silence_when_in_sync(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
|
||
_make_task("done", branch="feature/ET-010-done", wi="ET-010", age_s=3600)
|
||
fresh = _make_task("development", branch="feature/ET-011-fresh",
|
||
wi="ET-011", age_s=5)
|
||
busy = _make_task("development", branch="feature/ET-012-busy",
|
||
wi="ET-012", age_s=3600)
|
||
enqueue_job("reviewer", "enduro-trails", task_id=busy)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
spy.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
assert _stage_of(fresh) == "development"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-08 (AC-16): F-1 never advances the human analysis gate.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc08_analysis_not_advanced_by_f1(monkeypatch):
|
||
# Even if the analysis gate would "pass", F-1 must not touch analysis.
|
||
monkeypatch.setitem(
|
||
stage_engine.QG_CHECKS, "check_analysis_approved",
|
||
MagicMock(return_value=(True, "approved")),
|
||
)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
|
||
task_id = _make_task("analysis", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "analysis"
|
||
spy.assert_not_called()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-09: never-raise — one task blowing up does not stop the others.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc09_never_raise_isolates_failure(monkeypatch):
|
||
calls = []
|
||
|
||
def boom(task_id, stage, repo, wi, branch):
|
||
calls.append(task_id)
|
||
raise RuntimeError("boom")
|
||
|
||
monkeypatch.setattr(reconciler_mod, "advance_if_gate_passed", boom)
|
||
|
||
t1 = _make_task("development", branch="feature/ET-020-a", wi="ET-020", age_s=3600)
|
||
t2 = _make_task("development", branch="feature/ET-021-b", wi="ET-021", age_s=3600)
|
||
|
||
# Must not raise despite both tasks raising inside advance_if_gate_passed.
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert set(calls) == {t1, t2} # both attempted
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-10: kill-switches.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc10_kill_switch_disables_gate(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", False)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
_green_ci(monkeypatch)
|
||
|
||
task_id = _make_task("development", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
spy.assert_not_called()
|
||
|
||
|
||
def test_tc10_plane_switch_mutes_only_f2(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", True)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_plane_enabled", False)
|
||
|
||
plane_pass = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod.Reconciler, "_reconcile_plane_project", plane_pass)
|
||
# F-2 muted -> reconcile_plane_once is a no-op.
|
||
Reconciler().reconcile_plane_once()
|
||
plane_pass.assert_not_called()
|
||
|
||
# F-1 still runs.
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
assert _stage_of(task_id) == "review"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-20: observability — explicit unblock log line + telegram (AC-12 / F-4).
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc20_unblock_logs_and_notifies(monkeypatch, caplog):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
|
||
_make_task("development", wi="ET-042", age_s=3600)
|
||
|
||
rec = Reconciler()
|
||
with caplog.at_level("INFO", logger="orchestrator.reconciler"):
|
||
rec.reconcile_gate_once()
|
||
|
||
# Exact AC-12 contract string.
|
||
assert "reconciler: ET-042 development разблокирована (потерян webhook)" in caplog.text
|
||
assert rec.unblocked_total == 1
|
||
assert rec.last_unblocked == "ET-042"
|
||
tg.assert_called_once()
|
||
|
||
|
||
def test_tc20_no_telegram_when_disabled(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", False)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
|
||
_make_task("development", wi="ET-043", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
tg.assert_not_called()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-21: restart-safe daemon thread — start/stop/idempotent start.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc21_daemon_thread_lifecycle(monkeypatch):
|
||
# Avoid any real work in the loop: disable both branches, big interval.
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", False)
|
||
rec = Reconciler(interval_s=60)
|
||
|
||
rec.start()
|
||
assert rec._thread is not None and rec._thread.is_alive()
|
||
first_thread = rec._thread
|
||
|
||
# Idempotent: a second start does not spawn a new thread.
|
||
rec.start()
|
||
assert rec._thread is first_thread
|
||
|
||
rec.stop(timeout=5.0)
|
||
assert not first_thread.is_alive()
|
||
|
||
|
||
# ===========================================================================
|
||
# ORCH-060: F-1 skips escalated (max developer retries) / Blocked / Needs Input
|
||
# ===========================================================================
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-01 (AC-1): escalated dev task (exactly MAX_DEVELOPER_RETRIES dev runs) at a
|
||
# green gate is NOT unblocked — stays development, no job, count 0.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_01_escalated_at_limit_skipped(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
assert _jobs_for(task_id, "reviewer") == []
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-02 (AC-2): more dev runs than the cap (4–5) -> also skipped (>= boundary).
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_02_over_limit_skipped(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES + 2)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-03 (AC-3): regression — retry < cap (here 2) still advances to review.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_03_under_limit_still_advances(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES - 1)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "review"
|
||
assert rec.unblocked_total == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-04 (AC-4): twins — one at the cap (skip), one at cap-1 (advance). Exactly
|
||
# one advances.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_04_boundary_exactly_one_advances(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
at_limit = _make_task("development", branch="feature/ET-200-a",
|
||
wi="ET-200", age_s=3600)
|
||
below = _make_task("development", branch="feature/ET-201-b",
|
||
wi="ET-201", age_s=3600)
|
||
_add_dev_runs(at_limit, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
_add_dev_runs(below, stage_engine.MAX_DEVELOPER_RETRIES - 1)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(at_limit) == "development" # skipped
|
||
assert _stage_of(below) == "review" # advanced
|
||
assert rec.unblocked_total == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-05 (AC-5): explicit Plane Blocked (retry < cap) -> skipped.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_05_blocked_skipped(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value=_BLOCKED_UUID),
|
||
)
|
||
task_id = _make_task("development", age_s=3600)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-06 (AC-6): explicit Plane Needs Input (retry < cap) -> skipped.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_06_needs_input_skipped(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value=_NEEDS_INPUT_UUID),
|
||
)
|
||
task_id = _make_task("development", age_s=3600)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-07 (AC-7): no spam — escalated task triggers no unblock log / telegram /
|
||
# QG-failure notification, across several ticks.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_07_escalated_no_spam(monkeypatch, caplog):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
|
||
task_id = _make_task("development", wi="ET-210", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
|
||
rec = Reconciler()
|
||
with caplog.at_level("INFO", logger="orchestrator.reconciler"):
|
||
for _ in range(3):
|
||
rec.reconcile_gate_once()
|
||
|
||
assert "разблокирована" not in caplog.text
|
||
tg.assert_not_called()
|
||
stage_engine.notify_qg_failure.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-08 (AC-8): the gate (check_ci_green) is NOT even evaluated for an escalated
|
||
# task — Guard 1 skips before the pre-evaluation.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_08_no_gate_call_on_escalated(monkeypatch):
|
||
ci = _green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
ci.assert_not_called()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-09 (AC-9): F-2 never replays Blocked / Needs Input — those states are not
|
||
# in the polled set, so the handlers are never invoked.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_09_f2_does_not_replay_blocked(monkeypatch):
|
||
states = {
|
||
"in_progress": "IP", "to_analyse": "IP", "approved": "AP", "rejected": "RJ",
|
||
"blocked": "BL", "needs_input": "NI",
|
||
}
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "get_project_states", MagicMock(return_value=states)
|
||
)
|
||
captured = {}
|
||
|
||
def fake_list(pid, state_uuids):
|
||
captured["states"] = list(state_uuids)
|
||
# Plane filters client-side to the requested states, so a Blocked /
|
||
# Needs Input issue is structurally excluded from the result.
|
||
return []
|
||
|
||
monkeypatch.setattr(reconciler_mod, "list_issues_by_state", fake_list)
|
||
hss = MagicMock()
|
||
hv = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "handle_status_start", hss)
|
||
monkeypatch.setattr(reconciler_mod, "handle_verdict", hv)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.projects, "PROJECTS",
|
||
[MagicMock(repo="enduro-trails", plane_project_id="P")],
|
||
)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_plane_once()
|
||
|
||
assert "BL" not in captured["states"]
|
||
assert "NI" not in captured["states"]
|
||
hss.assert_not_called()
|
||
hv.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-10 (AC-10): never-raise — a Guard 2 lookup that raises for one task is
|
||
# isolated (that task is conservatively skipped); a neighbour
|
||
# still advances and the tick does not blow up.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_10_guard2_never_raise(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
bad = _make_task("development", branch="feature/ET-220-bad",
|
||
wi="ET-220", age_s=3600)
|
||
ok = _make_task("development", branch="feature/ET-221-ok",
|
||
wi="ET-221", age_s=3600)
|
||
|
||
def flaky(issue_id, project_id):
|
||
if issue_id == "plane-ET-220":
|
||
raise RuntimeError("plane boom")
|
||
return "some-non-gated-state"
|
||
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state", MagicMock(side_effect=flaky)
|
||
)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once() # must not raise
|
||
|
||
assert _stage_of(bad) == "development" # conservative skip
|
||
assert _stage_of(ok) == "review" # neighbour advanced
|
||
assert rec.unblocked_total == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-11 (AC-11): the cutoff comes from MAX_DEVELOPER_RETRIES, not a literal 3.
|
||
# Patching the constant to 2 makes a 2-run task escalate (it would
|
||
# have advanced under a hardcoded 3).
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_11_limit_from_constant(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod, "MAX_DEVELOPER_RETRIES", 2)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, 2) # == patched cap -> skip
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# AC-10 extra: the sub-flag reconcile_skip_blocked_enabled=False mutes ONLY
|
||
# Guard 2 (a Blocked task would then be reconciled), while Guard 1
|
||
# (escalated) stays active.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_subflag_disables_only_guard2(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.settings, "reconcile_skip_blocked_enabled", False
|
||
)
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value=_BLOCKED_UUID),
|
||
)
|
||
# Guard 2 disabled -> a Blocked task with retry < cap advances again.
|
||
blocked = _make_task("development", branch="feature/ET-230-a",
|
||
wi="ET-230", age_s=3600)
|
||
# Guard 1 stays active regardless of the sub-flag.
|
||
escalated = _make_task("development", branch="feature/ET-231-b",
|
||
wi="ET-231", age_s=3600)
|
||
_add_dev_runs(escalated, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(blocked) == "review" # Guard 2 muted
|
||
assert _stage_of(escalated) == "development" # Guard 1 still skips
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ORCH-066 TC-21 (AC-20 / BR-13): Guard 2 skips the active orchestrator waits
|
||
# (Awaiting Deploy / Deploying / Monitoring after Deploy) ONLY when they are
|
||
# DISTINCT statuses — an aliased (enduro) project must NOT widen the skip-set.
|
||
# ---------------------------------------------------------------------------
|
||
def _guard2(monkeypatch, states, cur_state):
|
||
"""Drive _is_blocked_or_needs_input with a chosen project state map + the
|
||
issue's current Plane state uuid."""
|
||
monkeypatch.setattr(reconciler_mod, "get_project_states",
|
||
MagicMock(return_value=states))
|
||
monkeypatch.setattr(reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value=cur_state))
|
||
monkeypatch.setattr(
|
||
reconciler_mod.projects, "get_project_by_repo",
|
||
MagicMock(return_value=MagicMock(plane_project_id="proj-test")),
|
||
)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.settings, "reconcile_skip_blocked_enabled", True
|
||
)
|
||
task = {"id": 1, "repo": "orchestrator", "plane_id": "iss-1"}
|
||
return Reconciler()._is_blocked_or_needs_input(task)
|
||
|
||
|
||
# orchestrator has the three new statuses as DISTINCT UUIDs.
|
||
_DISTINCT_STATES = {
|
||
"backlog": "bl-u", "todo": "td-u", "in_progress": "ip-u", "in_review": "inrev-u",
|
||
"review": "rev-u", "architecture": "arch-u", "development": "dev-u",
|
||
"testing": "test-u", "approved": "appr-u", "rejected": "rej-u", "done": "done-u",
|
||
"blocked": "blocked-u", "needs_input": "ni-u",
|
||
"awaiting_deploy": "await-u", "deploying": "deploying-u", "monitoring": "monitor-u",
|
||
}
|
||
|
||
|
||
def test_tc21_guard2_skips_distinct_active_waits(monkeypatch):
|
||
# Each active-wait status (distinct UUID) -> skipped (not revived).
|
||
assert _guard2(monkeypatch, _DISTINCT_STATES, "await-u") is True
|
||
assert _guard2(monkeypatch, _DISTINCT_STATES, "deploying-u") is True
|
||
assert _guard2(monkeypatch, _DISTINCT_STATES, "monitor-u") is True
|
||
# Explicit human gates still skip.
|
||
assert _guard2(monkeypatch, _DISTINCT_STATES, "blocked-u") is True
|
||
assert _guard2(monkeypatch, _DISTINCT_STATES, "ni-u") is True
|
||
# A normal working state is NOT skipped (gets reconciled).
|
||
assert _guard2(monkeypatch, _DISTINCT_STATES, "ip-u") is False
|
||
|
||
|
||
def test_tc21_guard2_aliased_waits_do_not_widen_skipset(monkeypatch):
|
||
# enduro: the new keys alias onto base working statuses -> they must NOT make
|
||
# F-1 skip a genuinely In Progress / In Review / Done task (anti-regress).
|
||
aliased = {
|
||
"backlog": "bl-u", "todo": "td-u", "in_progress": "ip-u", "in_review": "inrev-u",
|
||
"review": "rev-u", "architecture": "arch-u", "development": "dev-u",
|
||
"testing": "test-u", "approved": "appr-u", "rejected": "rej-u", "done": "done-u",
|
||
"blocked": "blocked-u", "needs_input": "ni-u",
|
||
# aliased onto base UUIDs (project did not create dedicated statuses).
|
||
"awaiting_deploy": "inrev-u", "deploying": "ip-u", "monitoring": "done-u",
|
||
}
|
||
# In Progress / In Review / Done are base working states -> NOT skipped.
|
||
assert _guard2(monkeypatch, aliased, "ip-u") is False
|
||
assert _guard2(monkeypatch, aliased, "inrev-u") is False
|
||
assert _guard2(monkeypatch, aliased, "done-u") is False
|
||
# The explicit human gates still skip.
|
||
assert _guard2(monkeypatch, aliased, "blocked-u") is True
|
||
|
||
|
||
# ===========================================================================
|
||
# ORCH-086: terminal-skip + state_uuid dedup on the F-1 (gate-side) path.
|
||
# Closes the gap of ORCH-068 (which covered only F-2). The spurious
|
||
# "ET-002 ... разблокирована (потерян webhook)" notification for a task that is
|
||
# already terminal in Plane (but drifted in the orchestrator DB) is suppressed.
|
||
# ===========================================================================
|
||
def _plane_terminal(monkeypatch, *, state_uuid="done-uuid",
|
||
states=None, groups=None):
|
||
"""Make Plane report ``state_uuid`` as the issue's current state, with the
|
||
given {key->uuid} states and {uuid->group} groups maps."""
|
||
monkeypatch.setattr(reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value=state_uuid))
|
||
monkeypatch.setattr(reconciler_mod, "get_project_states",
|
||
MagicMock(return_value=states if states is not None
|
||
else {"done": "done-uuid"}))
|
||
monkeypatch.setattr(reconciler_mod, "get_project_state_groups",
|
||
MagicMock(return_value=groups if groups is not None
|
||
else {"done-uuid": "completed"}))
|
||
|
||
|
||
# --- TC-86-01 (AC-1) -------------------------------------------------------
|
||
def test_tc86_01_terminal_in_plane_not_unblocked(monkeypatch):
|
||
"""enduro task NOT-done in the DB but terminal in Plane (group=completed),
|
||
green gate: F-1 must NOT call _note_unblock / send_telegram — neither on a
|
||
normal tick nor on the first pass of a fresh Reconciler (clean dedup)."""
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
note = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod.Reconciler, "_note_unblock", note)
|
||
_plane_terminal(monkeypatch) # Plane says Done (group=completed)
|
||
|
||
task_id = _make_task("development", wi="ET-002", age_s=3600)
|
||
|
||
# Fresh Reconciler -> empty _unblock_dedup -> the "first pass after restart"
|
||
# symptom is exercised; the terminal-skip must fire regardless of dedup.
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development" # never advanced
|
||
note.assert_not_called()
|
||
tg.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
assert rec.skipped_terminal_total >= 1
|
||
|
||
|
||
# --- TC-86-02 (AC-2) -------------------------------------------------------
|
||
def test_tc86_02_terminal_skip_counter_no_advance(monkeypatch):
|
||
"""Terminal-skip bumps skipped_terminal_total and never reaches
|
||
advance_if_gate_passed."""
|
||
spy = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "advance_if_gate_passed", spy)
|
||
_plane_terminal(monkeypatch)
|
||
|
||
_make_task("development", wi="ET-002", age_s=3600)
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert rec.skipped_terminal_total == 1
|
||
spy.assert_not_called()
|
||
|
||
|
||
# --- TC-86-03 (AC-2 / R1) --------------------------------------------------
|
||
def test_tc86_03_terminal_by_group_cancelled(monkeypatch):
|
||
"""Terminal detection by Plane state GROUP works for cancelled too, and is
|
||
project-independent (group discriminator, not a per-project key)."""
|
||
spy = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "advance_if_gate_passed", spy)
|
||
_plane_terminal(
|
||
monkeypatch, state_uuid="cancel-uuid",
|
||
states={"done": "done-uuid", "cancelled": "cancel-uuid"},
|
||
groups={"cancel-uuid": "cancelled"},
|
||
)
|
||
|
||
_make_task("development", wi="ET-002", age_s=3600)
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert rec.skipped_terminal_total == 1
|
||
spy.assert_not_called()
|
||
|
||
|
||
# --- TC-86-04 (AC-2 / R1) --------------------------------------------------
|
||
def test_tc86_04_terminal_fallback_logical_key_empty_groups(monkeypatch):
|
||
"""Fallback when groups are unavailable ({}): terminality by the project's
|
||
logical done/cancelled key."""
|
||
spy = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "advance_if_gate_passed", spy)
|
||
_plane_terminal(
|
||
monkeypatch, state_uuid="done-key-uuid",
|
||
states={"done": "done-key-uuid", "cancelled": "cancel-key-uuid"},
|
||
groups={}, # group unknown -> logical-key fallback
|
||
)
|
||
|
||
_make_task("development", wi="ET-002", age_s=3600)
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert rec.skipped_terminal_total == 1
|
||
spy.assert_not_called()
|
||
|
||
|
||
# --- TC-86-05 (AC-2) -------------------------------------------------------
|
||
def test_tc86_05_terminal_by_db_stage_cancelled(monkeypatch):
|
||
"""DB-side terminal drift: a task with stage='cancelled' (NOT filtered by
|
||
get_active_tasks_for_reconcile, which only drops 'done') is skipped locally
|
||
without reaching _note_unblock / advance — and bumps skipped_terminal_total."""
|
||
spy = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "advance_if_gate_passed", spy)
|
||
note = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod.Reconciler, "_note_unblock", note)
|
||
# A networked resolve must not even be needed for the DB-side guard.
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(side_effect=AssertionError("must not hit Plane for DB-cancelled")),
|
||
)
|
||
|
||
_make_task("cancelled", wi="ET-002", age_s=3600)
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert rec.skipped_terminal_total == 1
|
||
spy.assert_not_called()
|
||
note.assert_not_called()
|
||
|
||
|
||
# --- TC-86-06 (AC-3) -------------------------------------------------------
|
||
def test_tc86_06_legit_unblock_passes_state_uuid(monkeypatch):
|
||
"""A legitimate unblock calls _note_unblock with a non-empty state_uuid; the
|
||
dedup guard stores issue_id -> state_uuid."""
|
||
_green_ci(monkeypatch)
|
||
# Default fixture: fetch_issue_state -> 'some-non-gated-state', groups {} ->
|
||
# not terminal, not blocked -> the task advances.
|
||
task_id = _make_task("development", wi="ET-300", age_s=3600)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "review"
|
||
assert rec.unblocked_total == 1
|
||
assert rec._unblock_dedup.get("ET-300") == "some-non-gated-state"
|
||
|
||
|
||
# --- TC-86-07 (AC-3) -------------------------------------------------------
|
||
def test_tc86_07_repeat_tick_deduped(monkeypatch):
|
||
"""A repeat F-1 tick for the same issue+state_uuid is suppressed by the dedup
|
||
guard: deduped_total += 1 and no second send_telegram."""
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
# advance "succeeds" but leaves the stage put, so each tick reaches
|
||
# _note_unblock again with the SAME resolved state_uuid.
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "advance_if_gate_passed",
|
||
MagicMock(return_value=MagicMock(advanced=True)),
|
||
)
|
||
|
||
_make_task("development", wi="ET-301", age_s=3600)
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once() # first: notifies
|
||
rec.reconcile_gate_once() # second: same issue+state -> deduped
|
||
|
||
assert tg.call_count == 1
|
||
assert rec.unblocked_total == 1
|
||
assert rec.deduped_total == 1
|
||
|
||
|
||
# --- TC-86-08 (AC-4, anti-regress) -----------------------------------------
|
||
def test_tc86_08_legit_unblock_still_notifies(monkeypatch):
|
||
"""A NON-terminal genuinely stuck task (working Plane status, past grace, no
|
||
active job, green gate) is STILL advanced and notifies exactly once."""
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
|
||
task_id = _make_task("development", wi="ET-302", age_s=3600)
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "review"
|
||
tg.assert_called_once()
|
||
assert rec.unblocked_total == 1
|
||
assert rec.skipped_terminal_total == 0
|
||
|
||
|
||
# --- TC-86-09 (AC-5, never-raise) ------------------------------------------
|
||
def test_tc86_09_never_raise_no_false_notify(monkeypatch):
|
||
"""An exception in the terminal-detect / fetch_issue_state path does not blow
|
||
up the tick AND does not produce a false unblock (conservative)."""
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(side_effect=RuntimeError("plane boom")),
|
||
)
|
||
|
||
task_id = _make_task("development", wi="ET-303", age_s=3600)
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once() # must not raise
|
||
|
||
# resolve failed -> state_uuid None -> not terminal, Guard 2 conservative skip.
|
||
assert _stage_of(task_id) == "development"
|
||
tg.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# --- TC-86-11 (AC-6) -------------------------------------------------------
|
||
def test_tc86_11_terminal_skip_independent_of_guard2_flag(monkeypatch):
|
||
"""reconcile_skip_blocked_enabled=False (Guard 2 escape hatch) does NOT
|
||
disable the unconditional terminal-skip: a terminal task is still skipped."""
|
||
monkeypatch.setattr(
|
||
reconciler_mod.settings, "reconcile_skip_blocked_enabled", False
|
||
)
|
||
spy = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "advance_if_gate_passed", spy)
|
||
_plane_terminal(monkeypatch) # group=completed
|
||
|
||
_make_task("development", wi="ET-304", age_s=3600)
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert rec.skipped_terminal_total == 1
|
||
spy.assert_not_called()
|