claude-bot
4db8276f98
Reconciler F-1 could not tell "stuck by a lost webhook" from "escalated:
max developer retries reached, waiting for a human". With CI green and a
reviewer that kept sending REQUEST_CHANGES up to the cap, every tick
re-unblocked development -> review -> rollback -> re-unblock (incident
ET-013, infinite bounce: wasted agent runs, Telegram spam, parasitic load
on the shared self-hosting instance).
Add two pre-gate guards in Reconciler._reconcile_gate_task (after the
existing analysis/no-gate/active-job/grace guards, before the gate
pre-evaluation), each an early silent return (no advance, no unblocked_total
increment, no notifications):
- Guard 1 (escalated, deterministic, no network, checked first):
developer_retry_count(task_id) >= MAX_DEVELOPER_RETRIES. Promote
stage_engine._developer_retry_count to public developer_retry_count
(single source of truth; private alias kept). Limit from the constant,
not a literal 3.
- Guard 2 (explicit human Plane gate, Variant A, no DB migration): new
never-raise plane_sync.fetch_issue_state + Reconciler._is_blocked_or_needs_input;
any error/None/unresolved project -> conservative skip. New sub-flag
ORCH_RECONCILE_SKIP_BLOCKED_ENABLED mutes only the networked Guard 2.
F-2 unchanged: Blocked/Needs Input are outside {in_progress, approved,
rejected} so they are never replayed (regression test added). DB schema,
STAGE_TRANSITIONS, QG_CHECKS, never-raise, analysis carve-out and
kill-switches untouched.
Refs: ORCH-060
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
683 lines
26 KiB
Python
683 lines
26 KiB
Python
"""ORCH-053: tests for the gate-side stuck-task reconciler (F-1) + lifecycle.
|
||
|
||
These cover the F-1 sweeper (``Reconciler.reconcile_gate_once``), the per-stage
|
||
grace / config (``grace_for_stage``), the no-spam guarantee, the analysis carve-
|
||
out (AC-16), never-raise isolation, the kill-switch, the unblock observability
|
||
(AC-12 / F-4) and the restart-safe daemon thread (AC-11).
|
||
|
||
Everything that touches the network (the quality gate, Plane sync, Telegram) is
|
||
mocked at the src.stage_engine / src.reconciler level so the reconciler runs
|
||
against a real isolated sqlite DB (same convention as test_stage_engine.py).
|
||
"""
|
||
|
||
import os
|
||
import tempfile
|
||
|
||
import pytest
|
||
|
||
# Isolated test DB (set BEFORE importing src.* so settings picks it up).
|
||
_test_db = os.path.join(tempfile.gettempdir(), "test_orchestrator_reconciler.db")
|
||
os.environ["ORCH_DB_PATH"] = _test_db
|
||
os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
|
||
os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
|
||
os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
|
||
|
||
from unittest.mock import MagicMock # noqa: E402
|
||
|
||
import src.db as _db # noqa: E402
|
||
from src.db import init_db, get_db, enqueue_job # noqa: E402
|
||
from src import stage_engine # noqa: E402
|
||
from src import reconciler as reconciler_mod # noqa: E402
|
||
from src.reconciler import Reconciler, grace_for_stage # noqa: E402
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixtures
|
||
# ---------------------------------------------------------------------------
|
||
@pytest.fixture(autouse=True)
|
||
def fresh_db(monkeypatch):
|
||
"""Fresh isolated DB per test."""
|
||
monkeypatch.setattr(_db.settings, "db_path", _test_db)
|
||
if os.path.exists(_test_db):
|
||
os.unlink(_test_db)
|
||
init_db()
|
||
yield
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def silence_side_effects(monkeypatch):
|
||
"""No-op every Plane/Telegram/notification side effect in the engine so the
|
||
real advance_stage runs deterministically and offline."""
|
||
for name in (
|
||
"notify_stage_change",
|
||
"notify_qg_failure",
|
||
"notify_approve_requested",
|
||
"notify_error",
|
||
"send_telegram",
|
||
"plane_notify_stage",
|
||
"plane_notify_qg",
|
||
"plane_add_comment",
|
||
"set_issue_in_review",
|
||
"set_issue_needs_input",
|
||
"set_issue_in_progress",
|
||
"set_issue_blocked",
|
||
"set_issue_done",
|
||
):
|
||
monkeypatch.setattr(stage_engine, name, MagicMock(), raising=False)
|
||
|
||
|
||
def _make_task(stage, *, repo="enduro-trails", branch="feature/ET-001-x",
|
||
wi="ET-001", age_s=None):
|
||
"""Insert a task; if age_s is given, backdate updated_at by that many secs."""
|
||
conn = get_db()
|
||
cur = conn.execute(
|
||
"INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
|
||
"VALUES (?, ?, ?, ?, ?)",
|
||
(f"plane-{wi}", wi, repo, branch, stage),
|
||
)
|
||
task_id = cur.lastrowid
|
||
if age_s is not None:
|
||
conn.execute(
|
||
"UPDATE tasks SET updated_at = datetime('now', ?) WHERE id = ?",
|
||
(f"-{int(age_s)} seconds", task_id),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
return task_id
|
||
|
||
|
||
def _stage_of(task_id):
|
||
conn = get_db()
|
||
row = conn.execute("SELECT stage FROM tasks WHERE id = ?", (task_id,)).fetchone()
|
||
conn.close()
|
||
return row["stage"]
|
||
|
||
|
||
def _jobs_for(task_id, agent=None):
|
||
conn = get_db()
|
||
if agent:
|
||
rows = conn.execute(
|
||
"SELECT * FROM jobs WHERE task_id = ? AND agent = ?", (task_id, agent)
|
||
).fetchall()
|
||
else:
|
||
rows = conn.execute(
|
||
"SELECT * FROM jobs WHERE task_id = ?", (task_id,)
|
||
).fetchall()
|
||
conn.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def _green_ci(monkeypatch, value=(True, "CI green")):
|
||
"""Patch the check_ci_green entry in QG_CHECKS; return the mock."""
|
||
m = MagicMock(return_value=value)
|
||
monkeypatch.setitem(stage_engine.QG_CHECKS, "check_ci_green", m)
|
||
return m
|
||
|
||
|
||
# --- ORCH-060 fixtures / helpers -------------------------------------------
|
||
# State uuids the default "not blocked" fixture maps Blocked / Needs Input to.
|
||
_BLOCKED_UUID = "blocked-state-uuid"
|
||
_NEEDS_INPUT_UUID = "needs-input-state-uuid"
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def plane_state_not_blocked(monkeypatch):
|
||
"""ORCH-060 Guard 2 boundary: by default Plane says the issue is NOT in a
|
||
human gate, so the F-1 happy path runs deterministically offline (no real
|
||
httpx call). Tests that exercise Guard 2 override ``fetch_issue_state`` to
|
||
return ``_BLOCKED_UUID`` / ``_NEEDS_INPUT_UUID`` (or raise)."""
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value="some-non-gated-state"),
|
||
)
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "get_project_states",
|
||
MagicMock(return_value={
|
||
"blocked": _BLOCKED_UUID,
|
||
"needs_input": _NEEDS_INPUT_UUID,
|
||
}),
|
||
)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.projects, "get_project_by_repo",
|
||
MagicMock(return_value=MagicMock(plane_project_id="proj-test")),
|
||
)
|
||
|
||
|
||
def _add_dev_runs(task_id, n, agent="developer"):
|
||
"""Model N developer retries by inserting N agent_runs rows (ORCH-060)."""
|
||
conn = get_db()
|
||
for _ in range(n):
|
||
conn.execute(
|
||
"INSERT INTO agent_runs (task_id, agent) VALUES (?, ?)",
|
||
(task_id, agent),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-01: happy path — stuck development task is advanced to review
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc01_advances_stuck_development_task(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600) # well past grace
|
||
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "review"
|
||
reviewer_jobs = _jobs_for(task_id, "reviewer")
|
||
assert len(reviewer_jobs) == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-02: source of truth is the gate — advance goes through advance_stage
|
||
# with finished_agent=None (no own update_task_stage/enqueue_job).
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc02_advances_via_advance_stage_finished_agent_none(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
# advance_if_gate_passed resolves advance_stage as a module global.
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
|
||
task_id = _make_task("development", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert spy.call_count == 1
|
||
# finished_agent must be None (the webhook path).
|
||
_args, kwargs = spy.call_args
|
||
assert kwargs.get("finished_agent", "MISSING") is None
|
||
assert spy.call_args.args[0] == task_id
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-03: task with an active job is skipped — gate not evaluated, no advance.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc03_active_job_skipped(monkeypatch):
|
||
ci = _green_ci(monkeypatch)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
|
||
task_id = _make_task("development", age_s=3600)
|
||
enqueue_job("reviewer", "enduro-trails", task_id=task_id) # active (queued)
|
||
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
ci.assert_not_called()
|
||
spy.assert_not_called()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-04: per-stage grace — fresh task untouched, at-threshold task eligible.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc04_grace_boundary(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600)
|
||
_green_ci(monkeypatch)
|
||
|
||
fresh = _make_task("development", branch="feature/ET-002-fresh",
|
||
wi="ET-002", age_s=10) # < grace -> untouched
|
||
stuck = _make_task("development", branch="feature/ET-003-stuck",
|
||
wi="ET-003", age_s=3600) # >= grace -> advanced
|
||
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(fresh) == "development"
|
||
assert _stage_of(stuck) == "review"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-05: grace_for_stage reads overrides JSON; bad JSON -> default, no crash.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc05_grace_for_stage_overrides(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.settings,
|
||
"reconcile_grace_overrides_json",
|
||
'{"development": 30, "review": 7200}',
|
||
)
|
||
assert grace_for_stage("development") == 30
|
||
assert grace_for_stage("review") == 7200
|
||
# missing key -> default
|
||
assert grace_for_stage("testing") == 600
|
||
|
||
|
||
def test_tc05_grace_for_stage_invalid_json_falls_back(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.settings, "reconcile_grace_overrides_json", "{not valid json"
|
||
)
|
||
# Must not raise, must fall back to the default.
|
||
assert grace_for_stage("development") == 600
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-06: no spam — a stable-red gate never advances and never notifies, even
|
||
# across many ticks.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc06_red_gate_no_spam(monkeypatch):
|
||
_green_ci(monkeypatch, value=(False, "CI red"))
|
||
task_id = _make_task("development", age_s=3600)
|
||
|
||
rec = Reconciler()
|
||
for _ in range(5):
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
# The QG-failure notification branch inside advance_stage must never fire,
|
||
# because advance_if_gate_passed returns None on a red gate (no advance call).
|
||
stage_engine.notify_qg_failure.assert_not_called()
|
||
stage_engine.plane_notify_qg.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-07: silence when in sync — done / busy / within-grace tasks => no advance.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc07_silence_when_in_sync(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
|
||
_make_task("done", branch="feature/ET-010-done", wi="ET-010", age_s=3600)
|
||
fresh = _make_task("development", branch="feature/ET-011-fresh",
|
||
wi="ET-011", age_s=5)
|
||
busy = _make_task("development", branch="feature/ET-012-busy",
|
||
wi="ET-012", age_s=3600)
|
||
enqueue_job("reviewer", "enduro-trails", task_id=busy)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
spy.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
assert _stage_of(fresh) == "development"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-08 (AC-16): F-1 never advances the human analysis gate.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc08_analysis_not_advanced_by_f1(monkeypatch):
|
||
# Even if the analysis gate would "pass", F-1 must not touch analysis.
|
||
monkeypatch.setitem(
|
||
stage_engine.QG_CHECKS, "check_analysis_approved",
|
||
MagicMock(return_value=(True, "approved")),
|
||
)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
|
||
task_id = _make_task("analysis", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "analysis"
|
||
spy.assert_not_called()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-09: never-raise — one task blowing up does not stop the others.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc09_never_raise_isolates_failure(monkeypatch):
|
||
calls = []
|
||
|
||
def boom(task_id, stage, repo, wi, branch):
|
||
calls.append(task_id)
|
||
raise RuntimeError("boom")
|
||
|
||
monkeypatch.setattr(reconciler_mod, "advance_if_gate_passed", boom)
|
||
|
||
t1 = _make_task("development", branch="feature/ET-020-a", wi="ET-020", age_s=3600)
|
||
t2 = _make_task("development", branch="feature/ET-021-b", wi="ET-021", age_s=3600)
|
||
|
||
# Must not raise despite both tasks raising inside advance_if_gate_passed.
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert set(calls) == {t1, t2} # both attempted
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-10: kill-switches.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc10_kill_switch_disables_gate(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", False)
|
||
spy = MagicMock(wraps=stage_engine.advance_stage)
|
||
monkeypatch.setattr(stage_engine, "advance_stage", spy)
|
||
_green_ci(monkeypatch)
|
||
|
||
task_id = _make_task("development", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
spy.assert_not_called()
|
||
|
||
|
||
def test_tc10_plane_switch_mutes_only_f2(monkeypatch):
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", True)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_plane_enabled", False)
|
||
|
||
plane_pass = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod.Reconciler, "_reconcile_plane_project", plane_pass)
|
||
# F-2 muted -> reconcile_plane_once is a no-op.
|
||
Reconciler().reconcile_plane_once()
|
||
plane_pass.assert_not_called()
|
||
|
||
# F-1 still runs.
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
assert _stage_of(task_id) == "review"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-20: observability — explicit unblock log line + telegram (AC-12 / F-4).
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc20_unblock_logs_and_notifies(monkeypatch, caplog):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
|
||
_make_task("development", wi="ET-042", age_s=3600)
|
||
|
||
rec = Reconciler()
|
||
with caplog.at_level("INFO", logger="orchestrator.reconciler"):
|
||
rec.reconcile_gate_once()
|
||
|
||
# Exact AC-12 contract string.
|
||
assert "reconciler: ET-042 development разблокирована (потерян webhook)" in caplog.text
|
||
assert rec.unblocked_total == 1
|
||
assert rec.last_unblocked == "ET-042"
|
||
tg.assert_called_once()
|
||
|
||
|
||
def test_tc20_no_telegram_when_disabled(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", False)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
|
||
_make_task("development", wi="ET-043", age_s=3600)
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
tg.assert_not_called()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-21: restart-safe daemon thread — start/stop/idempotent start.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc21_daemon_thread_lifecycle(monkeypatch):
|
||
# Avoid any real work in the loop: disable both branches, big interval.
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", False)
|
||
rec = Reconciler(interval_s=60)
|
||
|
||
rec.start()
|
||
assert rec._thread is not None and rec._thread.is_alive()
|
||
first_thread = rec._thread
|
||
|
||
# Idempotent: a second start does not spawn a new thread.
|
||
rec.start()
|
||
assert rec._thread is first_thread
|
||
|
||
rec.stop(timeout=5.0)
|
||
assert not first_thread.is_alive()
|
||
|
||
|
||
# ===========================================================================
|
||
# ORCH-060: F-1 skips escalated (max developer retries) / Blocked / Needs Input
|
||
# ===========================================================================
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-01 (AC-1): escalated dev task (exactly MAX_DEVELOPER_RETRIES dev runs) at a
|
||
# green gate is NOT unblocked — stays development, no job, count 0.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_01_escalated_at_limit_skipped(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
assert _jobs_for(task_id, "reviewer") == []
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-02 (AC-2): more dev runs than the cap (4–5) -> also skipped (>= boundary).
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_02_over_limit_skipped(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES + 2)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-03 (AC-3): regression — retry < cap (here 2) still advances to review.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_03_under_limit_still_advances(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES - 1)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "review"
|
||
assert rec.unblocked_total == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-04 (AC-4): twins — one at the cap (skip), one at cap-1 (advance). Exactly
|
||
# one advances.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_04_boundary_exactly_one_advances(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
at_limit = _make_task("development", branch="feature/ET-200-a",
|
||
wi="ET-200", age_s=3600)
|
||
below = _make_task("development", branch="feature/ET-201-b",
|
||
wi="ET-201", age_s=3600)
|
||
_add_dev_runs(at_limit, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
_add_dev_runs(below, stage_engine.MAX_DEVELOPER_RETRIES - 1)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(at_limit) == "development" # skipped
|
||
assert _stage_of(below) == "review" # advanced
|
||
assert rec.unblocked_total == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-05 (AC-5): explicit Plane Blocked (retry < cap) -> skipped.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_05_blocked_skipped(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value=_BLOCKED_UUID),
|
||
)
|
||
task_id = _make_task("development", age_s=3600)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-06 (AC-6): explicit Plane Needs Input (retry < cap) -> skipped.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_06_needs_input_skipped(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value=_NEEDS_INPUT_UUID),
|
||
)
|
||
task_id = _make_task("development", age_s=3600)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-07 (AC-7): no spam — escalated task triggers no unblock log / telegram /
|
||
# QG-failure notification, across several ticks.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_07_escalated_no_spam(monkeypatch, caplog):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True)
|
||
tg = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
|
||
|
||
task_id = _make_task("development", wi="ET-210", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
|
||
rec = Reconciler()
|
||
with caplog.at_level("INFO", logger="orchestrator.reconciler"):
|
||
for _ in range(3):
|
||
rec.reconcile_gate_once()
|
||
|
||
assert "разблокирована" not in caplog.text
|
||
tg.assert_not_called()
|
||
stage_engine.notify_qg_failure.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-08 (AC-8): the gate (check_ci_green) is NOT even evaluated for an escalated
|
||
# task — Guard 1 skips before the pre-evaluation.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_08_no_gate_call_on_escalated(monkeypatch):
|
||
ci = _green_ci(monkeypatch)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
|
||
Reconciler().reconcile_gate_once()
|
||
|
||
ci.assert_not_called()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-09 (AC-9): F-2 never replays Blocked / Needs Input — those states are not
|
||
# in the polled set, so the handlers are never invoked.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_09_f2_does_not_replay_blocked(monkeypatch):
|
||
states = {
|
||
"in_progress": "IP", "approved": "AP", "rejected": "RJ",
|
||
"blocked": "BL", "needs_input": "NI",
|
||
}
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "get_project_states", MagicMock(return_value=states)
|
||
)
|
||
captured = {}
|
||
|
||
def fake_list(pid, state_uuids):
|
||
captured["states"] = list(state_uuids)
|
||
# Plane filters client-side to the requested states, so a Blocked /
|
||
# Needs Input issue is structurally excluded from the result.
|
||
return []
|
||
|
||
monkeypatch.setattr(reconciler_mod, "list_issues_by_state", fake_list)
|
||
hss = MagicMock()
|
||
hv = MagicMock()
|
||
monkeypatch.setattr(reconciler_mod, "handle_status_start", hss)
|
||
monkeypatch.setattr(reconciler_mod, "handle_verdict", hv)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.projects, "PROJECTS",
|
||
[MagicMock(repo="enduro-trails", plane_project_id="P")],
|
||
)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_plane_once()
|
||
|
||
assert "BL" not in captured["states"]
|
||
assert "NI" not in captured["states"]
|
||
hss.assert_not_called()
|
||
hv.assert_not_called()
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-10 (AC-10): never-raise — a Guard 2 lookup that raises for one task is
|
||
# isolated (that task is conservatively skipped); a neighbour
|
||
# still advances and the tick does not blow up.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_10_guard2_never_raise(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
bad = _make_task("development", branch="feature/ET-220-bad",
|
||
wi="ET-220", age_s=3600)
|
||
ok = _make_task("development", branch="feature/ET-221-ok",
|
||
wi="ET-221", age_s=3600)
|
||
|
||
def flaky(issue_id, project_id):
|
||
if issue_id == "plane-ET-220":
|
||
raise RuntimeError("plane boom")
|
||
return "some-non-gated-state"
|
||
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state", MagicMock(side_effect=flaky)
|
||
)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once() # must not raise
|
||
|
||
assert _stage_of(bad) == "development" # conservative skip
|
||
assert _stage_of(ok) == "review" # neighbour advanced
|
||
assert rec.unblocked_total == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# TC-11 (AC-11): the cutoff comes from MAX_DEVELOPER_RETRIES, not a literal 3.
|
||
# Patching the constant to 2 makes a 2-run task escalate (it would
|
||
# have advanced under a hardcoded 3).
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_11_limit_from_constant(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(reconciler_mod, "MAX_DEVELOPER_RETRIES", 2)
|
||
task_id = _make_task("development", age_s=3600)
|
||
_add_dev_runs(task_id, 2) # == patched cap -> skip
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(task_id) == "development"
|
||
assert rec.unblocked_total == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# AC-10 extra: the sub-flag reconcile_skip_blocked_enabled=False mutes ONLY
|
||
# Guard 2 (a Blocked task would then be reconciled), while Guard 1
|
||
# (escalated) stays active.
|
||
# ---------------------------------------------------------------------------
|
||
def test_tc060_subflag_disables_only_guard2(monkeypatch):
|
||
_green_ci(monkeypatch)
|
||
monkeypatch.setattr(
|
||
reconciler_mod.settings, "reconcile_skip_blocked_enabled", False
|
||
)
|
||
monkeypatch.setattr(
|
||
reconciler_mod, "fetch_issue_state",
|
||
MagicMock(return_value=_BLOCKED_UUID),
|
||
)
|
||
# Guard 2 disabled -> a Blocked task with retry < cap advances again.
|
||
blocked = _make_task("development", branch="feature/ET-230-a",
|
||
wi="ET-230", age_s=3600)
|
||
# Guard 1 stays active regardless of the sub-flag.
|
||
escalated = _make_task("development", branch="feature/ET-231-b",
|
||
wi="ET-231", age_s=3600)
|
||
_add_dev_runs(escalated, stage_engine.MAX_DEVELOPER_RETRIES)
|
||
|
||
rec = Reconciler()
|
||
rec.reconcile_gate_once()
|
||
|
||
assert _stage_of(blocked) == "review" # Guard 2 muted
|
||
assert _stage_of(escalated) == "development" # Guard 1 still skips
|