"""ORCH-053: tests for the gate-side stuck-task reconciler (F-1) + lifecycle. These cover the F-1 sweeper (``Reconciler.reconcile_gate_once``), the per-stage grace / config (``grace_for_stage``), the no-spam guarantee, the analysis carve- out (AC-16), never-raise isolation, the kill-switch, the unblock observability (AC-12 / F-4) and the restart-safe daemon thread (AC-11). Everything that touches the network (the quality gate, Plane sync, Telegram) is mocked at the src.stage_engine / src.reconciler level so the reconciler runs against a real isolated sqlite DB (same convention as test_stage_engine.py). """ import os import tempfile import pytest # Isolated test DB (set BEFORE importing src.* so settings picks it up). _test_db = os.path.join(tempfile.gettempdir(), "test_orchestrator_reconciler.db") os.environ["ORCH_DB_PATH"] = _test_db os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") from unittest.mock import MagicMock # noqa: E402 import src.db as _db # noqa: E402 from src.db import init_db, get_db, enqueue_job # noqa: E402 from src import stage_engine # noqa: E402 from src import reconciler as reconciler_mod # noqa: E402 from src.reconciler import Reconciler, grace_for_stage # noqa: E402 # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @pytest.fixture(autouse=True) def fresh_db(monkeypatch): """Fresh isolated DB per test.""" monkeypatch.setattr(_db.settings, "db_path", _test_db) if os.path.exists(_test_db): os.unlink(_test_db) init_db() yield @pytest.fixture(autouse=True) def silence_side_effects(monkeypatch): """No-op every Plane/Telegram/notification side effect in the engine so the real advance_stage runs deterministically and offline.""" for name in ( "notify_stage_change", "notify_qg_failure", "notify_approve_requested", "notify_error", "send_telegram", "plane_notify_stage", "plane_notify_qg", "plane_add_comment", "set_issue_in_review", "set_issue_needs_input", "set_issue_in_progress", "set_issue_blocked", "set_issue_done", ): monkeypatch.setattr(stage_engine, name, MagicMock(), raising=False) def _make_task(stage, *, repo="enduro-trails", branch="feature/ET-001-x", wi="ET-001", age_s=None): """Insert a task; if age_s is given, backdate updated_at by that many secs.""" conn = get_db() cur = conn.execute( "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " "VALUES (?, ?, ?, ?, ?)", (f"plane-{wi}", wi, repo, branch, stage), ) task_id = cur.lastrowid if age_s is not None: conn.execute( "UPDATE tasks SET updated_at = datetime('now', ?) WHERE id = ?", (f"-{int(age_s)} seconds", task_id), ) conn.commit() conn.close() return task_id def _stage_of(task_id): conn = get_db() row = conn.execute("SELECT stage FROM tasks WHERE id = ?", (task_id,)).fetchone() conn.close() return row["stage"] def _jobs_for(task_id, agent=None): conn = get_db() if agent: rows = conn.execute( "SELECT * FROM jobs WHERE task_id = ? AND agent = ?", (task_id, agent) ).fetchall() else: rows = conn.execute( "SELECT * FROM jobs WHERE task_id = ?", (task_id,) ).fetchall() conn.close() return [dict(r) for r in rows] def _green_ci(monkeypatch, value=(True, "CI green")): """Patch the check_ci_green entry in QG_CHECKS; return the mock.""" m = MagicMock(return_value=value) monkeypatch.setitem(stage_engine.QG_CHECKS, "check_ci_green", m) return m # --- ORCH-060 fixtures / helpers ------------------------------------------- # State uuids the default "not blocked" fixture maps Blocked / Needs Input to. _BLOCKED_UUID = "blocked-state-uuid" _NEEDS_INPUT_UUID = "needs-input-state-uuid" @pytest.fixture(autouse=True) def plane_state_not_blocked(monkeypatch): """ORCH-060 Guard 2 boundary: by default Plane says the issue is NOT in a human gate, so the F-1 happy path runs deterministically offline (no real httpx call). Tests that exercise Guard 2 override ``fetch_issue_state`` to return ``_BLOCKED_UUID`` / ``_NEEDS_INPUT_UUID`` (or raise).""" monkeypatch.setattr( reconciler_mod, "fetch_issue_state", MagicMock(return_value="some-non-gated-state"), ) monkeypatch.setattr( reconciler_mod, "get_project_states", MagicMock(return_value={ "blocked": _BLOCKED_UUID, "needs_input": _NEEDS_INPUT_UUID, }), ) monkeypatch.setattr( reconciler_mod.projects, "get_project_by_repo", MagicMock(return_value=MagicMock(plane_project_id="proj-test")), ) def _add_dev_runs(task_id, n, agent="developer"): """Model N developer retries by inserting N agent_runs rows (ORCH-060).""" conn = get_db() for _ in range(n): conn.execute( "INSERT INTO agent_runs (task_id, agent) VALUES (?, ?)", (task_id, agent), ) conn.commit() conn.close() # --------------------------------------------------------------------------- # TC-01: happy path — stuck development task is advanced to review # --------------------------------------------------------------------------- def test_tc01_advances_stuck_development_task(monkeypatch): _green_ci(monkeypatch) task_id = _make_task("development", age_s=3600) # well past grace Reconciler().reconcile_gate_once() assert _stage_of(task_id) == "review" reviewer_jobs = _jobs_for(task_id, "reviewer") assert len(reviewer_jobs) == 1 # --------------------------------------------------------------------------- # TC-02: source of truth is the gate — advance goes through advance_stage # with finished_agent=None (no own update_task_stage/enqueue_job). # --------------------------------------------------------------------------- def test_tc02_advances_via_advance_stage_finished_agent_none(monkeypatch): _green_ci(monkeypatch) spy = MagicMock(wraps=stage_engine.advance_stage) # advance_if_gate_passed resolves advance_stage as a module global. monkeypatch.setattr(stage_engine, "advance_stage", spy) task_id = _make_task("development", age_s=3600) Reconciler().reconcile_gate_once() assert spy.call_count == 1 # finished_agent must be None (the webhook path). _args, kwargs = spy.call_args assert kwargs.get("finished_agent", "MISSING") is None assert spy.call_args.args[0] == task_id # --------------------------------------------------------------------------- # TC-03: task with an active job is skipped — gate not evaluated, no advance. # --------------------------------------------------------------------------- def test_tc03_active_job_skipped(monkeypatch): ci = _green_ci(monkeypatch) spy = MagicMock(wraps=stage_engine.advance_stage) monkeypatch.setattr(stage_engine, "advance_stage", spy) task_id = _make_task("development", age_s=3600) enqueue_job("reviewer", "enduro-trails", task_id=task_id) # active (queued) Reconciler().reconcile_gate_once() assert _stage_of(task_id) == "development" ci.assert_not_called() spy.assert_not_called() # --------------------------------------------------------------------------- # TC-04: per-stage grace — fresh task untouched, at-threshold task eligible. # --------------------------------------------------------------------------- def test_tc04_grace_boundary(monkeypatch): monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600) _green_ci(monkeypatch) fresh = _make_task("development", branch="feature/ET-002-fresh", wi="ET-002", age_s=10) # < grace -> untouched stuck = _make_task("development", branch="feature/ET-003-stuck", wi="ET-003", age_s=3600) # >= grace -> advanced Reconciler().reconcile_gate_once() assert _stage_of(fresh) == "development" assert _stage_of(stuck) == "review" # --------------------------------------------------------------------------- # TC-05: grace_for_stage reads overrides JSON; bad JSON -> default, no crash. # --------------------------------------------------------------------------- def test_tc05_grace_for_stage_overrides(monkeypatch): monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600) monkeypatch.setattr( reconciler_mod.settings, "reconcile_grace_overrides_json", '{"development": 30, "review": 7200}', ) assert grace_for_stage("development") == 30 assert grace_for_stage("review") == 7200 # missing key -> default assert grace_for_stage("testing") == 600 def test_tc05_grace_for_stage_invalid_json_falls_back(monkeypatch): monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600) monkeypatch.setattr( reconciler_mod.settings, "reconcile_grace_overrides_json", "{not valid json" ) # Must not raise, must fall back to the default. assert grace_for_stage("development") == 600 # --------------------------------------------------------------------------- # TC-06: no spam — a stable-red gate never advances and never notifies, even # across many ticks. # --------------------------------------------------------------------------- def test_tc06_red_gate_no_spam(monkeypatch): _green_ci(monkeypatch, value=(False, "CI red")) task_id = _make_task("development", age_s=3600) rec = Reconciler() for _ in range(5): rec.reconcile_gate_once() assert _stage_of(task_id) == "development" # The QG-failure notification branch inside advance_stage must never fire, # because advance_if_gate_passed returns None on a red gate (no advance call). stage_engine.notify_qg_failure.assert_not_called() stage_engine.plane_notify_qg.assert_not_called() assert rec.unblocked_total == 0 # --------------------------------------------------------------------------- # TC-07: silence when in sync — done / busy / within-grace tasks => no advance. # --------------------------------------------------------------------------- def test_tc07_silence_when_in_sync(monkeypatch): _green_ci(monkeypatch) spy = MagicMock(wraps=stage_engine.advance_stage) monkeypatch.setattr(stage_engine, "advance_stage", spy) _make_task("done", branch="feature/ET-010-done", wi="ET-010", age_s=3600) fresh = _make_task("development", branch="feature/ET-011-fresh", wi="ET-011", age_s=5) busy = _make_task("development", branch="feature/ET-012-busy", wi="ET-012", age_s=3600) enqueue_job("reviewer", "enduro-trails", task_id=busy) rec = Reconciler() rec.reconcile_gate_once() spy.assert_not_called() assert rec.unblocked_total == 0 assert _stage_of(fresh) == "development" # --------------------------------------------------------------------------- # TC-08 (AC-16): F-1 never advances the human analysis gate. # --------------------------------------------------------------------------- def test_tc08_analysis_not_advanced_by_f1(monkeypatch): # Even if the analysis gate would "pass", F-1 must not touch analysis. monkeypatch.setitem( stage_engine.QG_CHECKS, "check_analysis_approved", MagicMock(return_value=(True, "approved")), ) spy = MagicMock(wraps=stage_engine.advance_stage) monkeypatch.setattr(stage_engine, "advance_stage", spy) task_id = _make_task("analysis", age_s=3600) Reconciler().reconcile_gate_once() assert _stage_of(task_id) == "analysis" spy.assert_not_called() # --------------------------------------------------------------------------- # TC-09: never-raise — one task blowing up does not stop the others. # --------------------------------------------------------------------------- def test_tc09_never_raise_isolates_failure(monkeypatch): calls = [] def boom(task_id, stage, repo, wi, branch): calls.append(task_id) raise RuntimeError("boom") monkeypatch.setattr(reconciler_mod, "advance_if_gate_passed", boom) t1 = _make_task("development", branch="feature/ET-020-a", wi="ET-020", age_s=3600) t2 = _make_task("development", branch="feature/ET-021-b", wi="ET-021", age_s=3600) # Must not raise despite both tasks raising inside advance_if_gate_passed. Reconciler().reconcile_gate_once() assert set(calls) == {t1, t2} # both attempted # --------------------------------------------------------------------------- # TC-10: kill-switches. # --------------------------------------------------------------------------- def test_tc10_kill_switch_disables_gate(monkeypatch): monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", False) spy = MagicMock(wraps=stage_engine.advance_stage) monkeypatch.setattr(stage_engine, "advance_stage", spy) _green_ci(monkeypatch) task_id = _make_task("development", age_s=3600) Reconciler().reconcile_gate_once() assert _stage_of(task_id) == "development" spy.assert_not_called() def test_tc10_plane_switch_mutes_only_f2(monkeypatch): monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", True) monkeypatch.setattr(reconciler_mod.settings, "reconcile_plane_enabled", False) plane_pass = MagicMock() monkeypatch.setattr(reconciler_mod.Reconciler, "_reconcile_plane_project", plane_pass) # F-2 muted -> reconcile_plane_once is a no-op. Reconciler().reconcile_plane_once() plane_pass.assert_not_called() # F-1 still runs. _green_ci(monkeypatch) task_id = _make_task("development", age_s=3600) Reconciler().reconcile_gate_once() assert _stage_of(task_id) == "review" # --------------------------------------------------------------------------- # TC-20: observability — explicit unblock log line + telegram (AC-12 / F-4). # --------------------------------------------------------------------------- def test_tc20_unblock_logs_and_notifies(monkeypatch, caplog): _green_ci(monkeypatch) monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True) tg = MagicMock() monkeypatch.setattr(reconciler_mod, "send_telegram", tg) _make_task("development", wi="ET-042", age_s=3600) rec = Reconciler() with caplog.at_level("INFO", logger="orchestrator.reconciler"): rec.reconcile_gate_once() # Exact AC-12 contract string. assert "reconciler: ET-042 development разблокирована (потерян webhook)" in caplog.text assert rec.unblocked_total == 1 assert rec.last_unblocked == "ET-042" tg.assert_called_once() def test_tc20_no_telegram_when_disabled(monkeypatch): _green_ci(monkeypatch) monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", False) tg = MagicMock() monkeypatch.setattr(reconciler_mod, "send_telegram", tg) _make_task("development", wi="ET-043", age_s=3600) Reconciler().reconcile_gate_once() tg.assert_not_called() # --------------------------------------------------------------------------- # TC-21: restart-safe daemon thread — start/stop/idempotent start. # --------------------------------------------------------------------------- def test_tc21_daemon_thread_lifecycle(monkeypatch): # Avoid any real work in the loop: disable both branches, big interval. monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", False) rec = Reconciler(interval_s=60) rec.start() assert rec._thread is not None and rec._thread.is_alive() first_thread = rec._thread # Idempotent: a second start does not spawn a new thread. rec.start() assert rec._thread is first_thread rec.stop(timeout=5.0) assert not first_thread.is_alive() # =========================================================================== # ORCH-060: F-1 skips escalated (max developer retries) / Blocked / Needs Input # =========================================================================== # --------------------------------------------------------------------------- # TC-01 (AC-1): escalated dev task (exactly MAX_DEVELOPER_RETRIES dev runs) at a # green gate is NOT unblocked — stays development, no job, count 0. # --------------------------------------------------------------------------- def test_tc060_01_escalated_at_limit_skipped(monkeypatch): _green_ci(monkeypatch) task_id = _make_task("development", age_s=3600) _add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES) rec = Reconciler() rec.reconcile_gate_once() assert _stage_of(task_id) == "development" assert rec.unblocked_total == 0 assert _jobs_for(task_id, "reviewer") == [] # --------------------------------------------------------------------------- # TC-02 (AC-2): more dev runs than the cap (4–5) -> also skipped (>= boundary). # --------------------------------------------------------------------------- def test_tc060_02_over_limit_skipped(monkeypatch): _green_ci(monkeypatch) task_id = _make_task("development", age_s=3600) _add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES + 2) rec = Reconciler() rec.reconcile_gate_once() assert _stage_of(task_id) == "development" assert rec.unblocked_total == 0 # --------------------------------------------------------------------------- # TC-03 (AC-3): regression — retry < cap (here 2) still advances to review. # --------------------------------------------------------------------------- def test_tc060_03_under_limit_still_advances(monkeypatch): _green_ci(monkeypatch) task_id = _make_task("development", age_s=3600) _add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES - 1) rec = Reconciler() rec.reconcile_gate_once() assert _stage_of(task_id) == "review" assert rec.unblocked_total == 1 # --------------------------------------------------------------------------- # TC-04 (AC-4): twins — one at the cap (skip), one at cap-1 (advance). Exactly # one advances. # --------------------------------------------------------------------------- def test_tc060_04_boundary_exactly_one_advances(monkeypatch): _green_ci(monkeypatch) at_limit = _make_task("development", branch="feature/ET-200-a", wi="ET-200", age_s=3600) below = _make_task("development", branch="feature/ET-201-b", wi="ET-201", age_s=3600) _add_dev_runs(at_limit, stage_engine.MAX_DEVELOPER_RETRIES) _add_dev_runs(below, stage_engine.MAX_DEVELOPER_RETRIES - 1) rec = Reconciler() rec.reconcile_gate_once() assert _stage_of(at_limit) == "development" # skipped assert _stage_of(below) == "review" # advanced assert rec.unblocked_total == 1 # --------------------------------------------------------------------------- # TC-05 (AC-5): explicit Plane Blocked (retry < cap) -> skipped. # --------------------------------------------------------------------------- def test_tc060_05_blocked_skipped(monkeypatch): _green_ci(monkeypatch) monkeypatch.setattr( reconciler_mod, "fetch_issue_state", MagicMock(return_value=_BLOCKED_UUID), ) task_id = _make_task("development", age_s=3600) rec = Reconciler() rec.reconcile_gate_once() assert _stage_of(task_id) == "development" assert rec.unblocked_total == 0 # --------------------------------------------------------------------------- # TC-06 (AC-6): explicit Plane Needs Input (retry < cap) -> skipped. # --------------------------------------------------------------------------- def test_tc060_06_needs_input_skipped(monkeypatch): _green_ci(monkeypatch) monkeypatch.setattr( reconciler_mod, "fetch_issue_state", MagicMock(return_value=_NEEDS_INPUT_UUID), ) task_id = _make_task("development", age_s=3600) rec = Reconciler() rec.reconcile_gate_once() assert _stage_of(task_id) == "development" assert rec.unblocked_total == 0 # --------------------------------------------------------------------------- # TC-07 (AC-7): no spam — escalated task triggers no unblock log / telegram / # QG-failure notification, across several ticks. # --------------------------------------------------------------------------- def test_tc060_07_escalated_no_spam(monkeypatch, caplog): _green_ci(monkeypatch) monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True) tg = MagicMock() monkeypatch.setattr(reconciler_mod, "send_telegram", tg) task_id = _make_task("development", wi="ET-210", age_s=3600) _add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES) rec = Reconciler() with caplog.at_level("INFO", logger="orchestrator.reconciler"): for _ in range(3): rec.reconcile_gate_once() assert "разблокирована" not in caplog.text tg.assert_not_called() stage_engine.notify_qg_failure.assert_not_called() assert rec.unblocked_total == 0 # --------------------------------------------------------------------------- # TC-08 (AC-8): the gate (check_ci_green) is NOT even evaluated for an escalated # task — Guard 1 skips before the pre-evaluation. # --------------------------------------------------------------------------- def test_tc060_08_no_gate_call_on_escalated(monkeypatch): ci = _green_ci(monkeypatch) task_id = _make_task("development", age_s=3600) _add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES) Reconciler().reconcile_gate_once() ci.assert_not_called() # --------------------------------------------------------------------------- # TC-09 (AC-9): F-2 never replays Blocked / Needs Input — those states are not # in the polled set, so the handlers are never invoked. # --------------------------------------------------------------------------- def test_tc060_09_f2_does_not_replay_blocked(monkeypatch): states = { "in_progress": "IP", "approved": "AP", "rejected": "RJ", "blocked": "BL", "needs_input": "NI", } monkeypatch.setattr( reconciler_mod, "get_project_states", MagicMock(return_value=states) ) captured = {} def fake_list(pid, state_uuids): captured["states"] = list(state_uuids) # Plane filters client-side to the requested states, so a Blocked / # Needs Input issue is structurally excluded from the result. return [] monkeypatch.setattr(reconciler_mod, "list_issues_by_state", fake_list) hss = MagicMock() hv = MagicMock() monkeypatch.setattr(reconciler_mod, "handle_status_start", hss) monkeypatch.setattr(reconciler_mod, "handle_verdict", hv) monkeypatch.setattr( reconciler_mod.projects, "PROJECTS", [MagicMock(repo="enduro-trails", plane_project_id="P")], ) rec = Reconciler() rec.reconcile_plane_once() assert "BL" not in captured["states"] assert "NI" not in captured["states"] hss.assert_not_called() hv.assert_not_called() assert rec.unblocked_total == 0 # --------------------------------------------------------------------------- # TC-10 (AC-10): never-raise — a Guard 2 lookup that raises for one task is # isolated (that task is conservatively skipped); a neighbour # still advances and the tick does not blow up. # --------------------------------------------------------------------------- def test_tc060_10_guard2_never_raise(monkeypatch): _green_ci(monkeypatch) bad = _make_task("development", branch="feature/ET-220-bad", wi="ET-220", age_s=3600) ok = _make_task("development", branch="feature/ET-221-ok", wi="ET-221", age_s=3600) def flaky(issue_id, project_id): if issue_id == "plane-ET-220": raise RuntimeError("plane boom") return "some-non-gated-state" monkeypatch.setattr( reconciler_mod, "fetch_issue_state", MagicMock(side_effect=flaky) ) rec = Reconciler() rec.reconcile_gate_once() # must not raise assert _stage_of(bad) == "development" # conservative skip assert _stage_of(ok) == "review" # neighbour advanced assert rec.unblocked_total == 1 # --------------------------------------------------------------------------- # TC-11 (AC-11): the cutoff comes from MAX_DEVELOPER_RETRIES, not a literal 3. # Patching the constant to 2 makes a 2-run task escalate (it would # have advanced under a hardcoded 3). # --------------------------------------------------------------------------- def test_tc060_11_limit_from_constant(monkeypatch): _green_ci(monkeypatch) monkeypatch.setattr(reconciler_mod, "MAX_DEVELOPER_RETRIES", 2) task_id = _make_task("development", age_s=3600) _add_dev_runs(task_id, 2) # == patched cap -> skip rec = Reconciler() rec.reconcile_gate_once() assert _stage_of(task_id) == "development" assert rec.unblocked_total == 0 # --------------------------------------------------------------------------- # AC-10 extra: the sub-flag reconcile_skip_blocked_enabled=False mutes ONLY # Guard 2 (a Blocked task would then be reconciled), while Guard 1 # (escalated) stays active. # --------------------------------------------------------------------------- def test_tc060_subflag_disables_only_guard2(monkeypatch): _green_ci(monkeypatch) monkeypatch.setattr( reconciler_mod.settings, "reconcile_skip_blocked_enabled", False ) monkeypatch.setattr( reconciler_mod, "fetch_issue_state", MagicMock(return_value=_BLOCKED_UUID), ) # Guard 2 disabled -> a Blocked task with retry < cap advances again. blocked = _make_task("development", branch="feature/ET-230-a", wi="ET-230", age_s=3600) # Guard 1 stays active regardless of the sub-flag. escalated = _make_task("development", branch="feature/ET-231-b", wi="ET-231", age_s=3600) _add_dev_runs(escalated, stage_engine.MAX_DEVELOPER_RETRIES) rec = Reconciler() rec.reconcile_gate_once() assert _stage_of(blocked) == "review" # Guard 2 muted assert _stage_of(escalated) == "development" # Guard 1 still skips