Merge remote-tracking branch 'origin/main' into feature/ORCH-036-orch-36-deploy-b

# Conflicts:
#	.env.example
#	CHANGELOG.md
#	docs/architecture/README.md
#	docs/operations/INFRA.md
#	src/config.py

tests/conftest.py

@@ -37,6 +37,10 @@ def _no_telegram(monkeypatch):
     monkeypatch.setattr("src.webhooks.plane.send_telegram", _noop, raising=False)
     monkeypatch.setattr("src.agents.launcher.send_telegram", _noop, raising=False)
     monkeypatch.setattr("src.queue_worker.send_telegram", _noop, raising=False)
+    # ORCH-053: the reconciler binds send_telegram as a MODULE-LEVEL name
+    # (from .notifications import send_telegram), so the source patch alone would
+    # not intercept its unblock notification — patch it here too.
+    monkeypatch.setattr("src.reconciler.send_telegram", _noop, raising=False)
     yield
 
 

tests/test_config.py

@@ -72,3 +72,46 @@ def test_merge_gate_settings_env_override(monkeypatch):
     assert s.merge_lock_timeout_s == 90
     assert s.merge_defer_delay_s == 5
     assert s.merge_defer_max_attempts == 9
+
+
+# ---------------------------------------------------------------------------
+# ORCH-053 / TC-22: reconcile_* settings defaults + env override.
+# ---------------------------------------------------------------------------
+_RECONCILE_ENV = (
+    "ORCH_RECONCILE_ENABLED",
+    "ORCH_RECONCILE_INTERVAL_S",
+    "ORCH_RECONCILE_PLANE_ENABLED",
+    "ORCH_RECONCILE_GRACE_DEFAULT_S",
+    "ORCH_RECONCILE_GRACE_OVERRIDES_JSON",
+    "ORCH_RECONCILE_NOTIFY_UNBLOCK",
+)
+
+
+def test_reconcile_settings_defaults(monkeypatch):
+    """TC-22 / AC-13: documented defaults when no env is set."""
+    for name in _RECONCILE_ENV:
+        monkeypatch.delenv(name, raising=False)
+    s = Settings()
+    assert s.reconcile_enabled is True
+    assert s.reconcile_interval_s == 120
+    assert s.reconcile_plane_enabled is True
+    assert s.reconcile_grace_default_s == 600
+    assert s.reconcile_grace_overrides_json == ""
+    assert s.reconcile_notify_unblock is True
+
+
+def test_reconcile_settings_env_override(monkeypatch):
+    """TC-22 / AC-13: each field is read from its ORCH_* env var."""
+    monkeypatch.setenv("ORCH_RECONCILE_ENABLED", "false")
+    monkeypatch.setenv("ORCH_RECONCILE_INTERVAL_S", "300")
+    monkeypatch.setenv("ORCH_RECONCILE_PLANE_ENABLED", "false")
+    monkeypatch.setenv("ORCH_RECONCILE_GRACE_DEFAULT_S", "900")
+    monkeypatch.setenv("ORCH_RECONCILE_GRACE_OVERRIDES_JSON", '{"development": 300}')
+    monkeypatch.setenv("ORCH_RECONCILE_NOTIFY_UNBLOCK", "false")
+    s = Settings()
+    assert s.reconcile_enabled is False
+    assert s.reconcile_interval_s == 300
+    assert s.reconcile_plane_enabled is False
+    assert s.reconcile_grace_default_s == 900
+    assert s.reconcile_grace_overrides_json == '{"development": 300}'
+    assert s.reconcile_notify_unblock is False

tests/test_gitea_sha_resolve.py

@@ -0,0 +1,119 @@
+"""ORCH-053 (F-3): sha->branch resolution hardening in handle_ci_status.
+
+When a CI-status webhook carries no ``branches[]`` and the SHA cannot be
+resolved to a feature branch via ``git branch -r --contains`` (lost on a 502
+rebuild, shallow clone, etc.), handle_ci_status now falls back to the tasks DB
+and matches the UNIQUE development-stage task of the repo. Ambiguity (more than
+one development task) is deliberately left unresolved so it can never make a
+false match.
+
+The git subprocess and the network QG / Plane / Telegram side effects are mocked
+so the handler runs offline against a real isolated sqlite DB.
+"""
+
+import asyncio
+import os
+import tempfile
+from types import SimpleNamespace
+
+import pytest
+
+_test_db = os.path.join(tempfile.gettempdir(), "test_orchestrator_gitea_sha.db")
+os.environ["ORCH_DB_PATH"] = _test_db
+os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
+os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
+os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
+
+from unittest.mock import MagicMock  # noqa: E402
+
+import src.db as _db  # noqa: E402
+from src.db import init_db, get_db  # noqa: E402
+from src.webhooks import gitea as gitea_mod  # noqa: E402
+
+
+@pytest.fixture(autouse=True)
+def fresh_db(monkeypatch):
+    monkeypatch.setattr(_db.settings, "db_path", _test_db)
+    if os.path.exists(_test_db):
+        os.unlink(_test_db)
+    init_db()
+    yield
+
+
+@pytest.fixture(autouse=True)
+def silence_and_stub_git(monkeypatch):
+    # git branch -r --contains <sha> resolves to nothing (forces the DB fallback).
+    monkeypatch.setattr(
+        gitea_mod.subprocess, "run",
+        lambda *a, **k: SimpleNamespace(stdout="", returncode=0),
+    )
+    # Mute the network side effects bound module-level in gitea.
+    for name in ("notify_stage_change", "notify_qg_failure", "notify_error",
+                 "plane_notify_stage"):
+        monkeypatch.setattr(gitea_mod, name, MagicMock(), raising=False)
+
+
+def _make_dev_task(branch, wi, repo="enduro-trails"):
+    conn = get_db()
+    cur = conn.execute(
+        "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
+        "VALUES (?, ?, ?, ?, 'development')",
+        (f"plane-{wi}", wi, repo, branch),
+    )
+    tid = cur.lastrowid
+    conn.commit()
+    conn.close()
+    return tid
+
+
+def _stage_of(task_id):
+    conn = get_db()
+    row = conn.execute("SELECT stage FROM tasks WHERE id = ?", (task_id,)).fetchone()
+    conn.close()
+    return row["stage"]
+
+
+def _ci_payload(sha="deadbeef", repo="enduro-trails", state="success"):
+    return {
+        "state": state,
+        "sha": sha,
+        "branches": [],  # no branch in the event -> forces resolution
+        "repository": {"name": repo},
+    }
+
+
+# ---------------------------------------------------------------------------
+# TC-18: unique development task -> DB fallback resolves the branch, advances.
+# ---------------------------------------------------------------------------
+def test_tc18_db_fallback_unique_match_advances(monkeypatch):
+    ci = MagicMock(return_value=(True, "CI green"))
+    monkeypatch.setattr(gitea_mod, "check_ci_green", ci)
+
+    tid = _make_dev_task("feature/ET-050-x", "ET-050")
+
+    asyncio.run(gitea_mod.handle_ci_status(_ci_payload()))
+
+    assert _stage_of(tid) == "review"
+    ci.assert_called_once()
+    # The fallback resolved to the unique dev task's branch.
+    assert ci.call_args.args[1] == "feature/ET-050-x"
+
+
+# ---------------------------------------------------------------------------
+# TC-19: several development tasks -> ambiguous -> no false match, no advance.
+# ---------------------------------------------------------------------------
+def test_tc19_db_fallback_ambiguous_no_match(monkeypatch, caplog):
+    ci = MagicMock(return_value=(True, "CI green"))
+    monkeypatch.setattr(gitea_mod, "check_ci_green", ci)
+
+    t1 = _make_dev_task("feature/ET-051-a", "ET-051")
+    t2 = _make_dev_task("feature/ET-052-b", "ET-052")
+
+    with caplog.at_level("INFO", logger="orchestrator.webhooks.gitea"):
+        asyncio.run(gitea_mod.handle_ci_status(_ci_payload()))
+
+    # Ambiguity -> branch unresolved -> handler returns before touching the gate.
+    assert _stage_of(t1) == "development"
+    assert _stage_of(t2) == "development"
+    ci.assert_not_called()
+    assert "could not determine branch" in caplog.text

tests/test_reconciler.py

@@ -0,0 +1,379 @@
+"""ORCH-053: tests for the gate-side stuck-task reconciler (F-1) + lifecycle.
+
+These cover the F-1 sweeper (``Reconciler.reconcile_gate_once``), the per-stage
+grace / config (``grace_for_stage``), the no-spam guarantee, the analysis carve-
+out (AC-16), never-raise isolation, the kill-switch, the unblock observability
+(AC-12 / F-4) and the restart-safe daemon thread (AC-11).
+
+Everything that touches the network (the quality gate, Plane sync, Telegram) is
+mocked at the src.stage_engine / src.reconciler level so the reconciler runs
+against a real isolated sqlite DB (same convention as test_stage_engine.py).
+"""
+
+import os
+import tempfile
+
+import pytest
+
+# Isolated test DB (set BEFORE importing src.* so settings picks it up).
+_test_db = os.path.join(tempfile.gettempdir(), "test_orchestrator_reconciler.db")
+os.environ["ORCH_DB_PATH"] = _test_db
+os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
+os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
+os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
+
+from unittest.mock import MagicMock  # noqa: E402
+
+import src.db as _db  # noqa: E402
+from src.db import init_db, get_db, enqueue_job  # noqa: E402
+from src import stage_engine  # noqa: E402
+from src import reconciler as reconciler_mod  # noqa: E402
+from src.reconciler import Reconciler, grace_for_stage  # noqa: E402
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+@pytest.fixture(autouse=True)
+def fresh_db(monkeypatch):
+    """Fresh isolated DB per test."""
+    monkeypatch.setattr(_db.settings, "db_path", _test_db)
+    if os.path.exists(_test_db):
+        os.unlink(_test_db)
+    init_db()
+    yield
+
+
+@pytest.fixture(autouse=True)
+def silence_side_effects(monkeypatch):
+    """No-op every Plane/Telegram/notification side effect in the engine so the
+    real advance_stage runs deterministically and offline."""
+    for name in (
+        "notify_stage_change",
+        "notify_qg_failure",
+        "notify_approve_requested",
+        "notify_error",
+        "send_telegram",
+        "plane_notify_stage",
+        "plane_notify_qg",
+        "plane_add_comment",
+        "set_issue_in_review",
+        "set_issue_needs_input",
+        "set_issue_in_progress",
+        "set_issue_blocked",
+        "set_issue_done",
+    ):
+        monkeypatch.setattr(stage_engine, name, MagicMock(), raising=False)
+
+
+def _make_task(stage, *, repo="enduro-trails", branch="feature/ET-001-x",
+               wi="ET-001", age_s=None):
+    """Insert a task; if age_s is given, backdate updated_at by that many secs."""
+    conn = get_db()
+    cur = conn.execute(
+        "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
+        "VALUES (?, ?, ?, ?, ?)",
+        (f"plane-{wi}", wi, repo, branch, stage),
+    )
+    task_id = cur.lastrowid
+    if age_s is not None:
+        conn.execute(
+            "UPDATE tasks SET updated_at = datetime('now', ?) WHERE id = ?",
+            (f"-{int(age_s)} seconds", task_id),
+        )
+    conn.commit()
+    conn.close()
+    return task_id
+
+
+def _stage_of(task_id):
+    conn = get_db()
+    row = conn.execute("SELECT stage FROM tasks WHERE id = ?", (task_id,)).fetchone()
+    conn.close()
+    return row["stage"]
+
+
+def _jobs_for(task_id, agent=None):
+    conn = get_db()
+    if agent:
+        rows = conn.execute(
+            "SELECT * FROM jobs WHERE task_id = ? AND agent = ?", (task_id, agent)
+        ).fetchall()
+    else:
+        rows = conn.execute(
+            "SELECT * FROM jobs WHERE task_id = ?", (task_id,)
+        ).fetchall()
+    conn.close()
+    return [dict(r) for r in rows]
+
+
+def _green_ci(monkeypatch, value=(True, "CI green")):
+    """Patch the check_ci_green entry in QG_CHECKS; return the mock."""
+    m = MagicMock(return_value=value)
+    monkeypatch.setitem(stage_engine.QG_CHECKS, "check_ci_green", m)
+    return m
+
+
+# ---------------------------------------------------------------------------
+# TC-01: happy path — stuck development task is advanced to review
+# ---------------------------------------------------------------------------
+def test_tc01_advances_stuck_development_task(monkeypatch):
+    _green_ci(monkeypatch)
+    task_id = _make_task("development", age_s=3600)  # well past grace
+
+    Reconciler().reconcile_gate_once()
+
+    assert _stage_of(task_id) == "review"
+    reviewer_jobs = _jobs_for(task_id, "reviewer")
+    assert len(reviewer_jobs) == 1
+
+
+# ---------------------------------------------------------------------------
+# TC-02: source of truth is the gate — advance goes through advance_stage
+#        with finished_agent=None (no own update_task_stage/enqueue_job).
+# ---------------------------------------------------------------------------
+def test_tc02_advances_via_advance_stage_finished_agent_none(monkeypatch):
+    _green_ci(monkeypatch)
+    spy = MagicMock(wraps=stage_engine.advance_stage)
+    # advance_if_gate_passed resolves advance_stage as a module global.
+    monkeypatch.setattr(stage_engine, "advance_stage", spy)
+
+    task_id = _make_task("development", age_s=3600)
+    Reconciler().reconcile_gate_once()
+
+    assert spy.call_count == 1
+    # finished_agent must be None (the webhook path).
+    _args, kwargs = spy.call_args
+    assert kwargs.get("finished_agent", "MISSING") is None
+    assert spy.call_args.args[0] == task_id
+
+
+# ---------------------------------------------------------------------------
+# TC-03: task with an active job is skipped — gate not evaluated, no advance.
+# ---------------------------------------------------------------------------
+def test_tc03_active_job_skipped(monkeypatch):
+    ci = _green_ci(monkeypatch)
+    spy = MagicMock(wraps=stage_engine.advance_stage)
+    monkeypatch.setattr(stage_engine, "advance_stage", spy)
+
+    task_id = _make_task("development", age_s=3600)
+    enqueue_job("reviewer", "enduro-trails", task_id=task_id)  # active (queued)
+
+    Reconciler().reconcile_gate_once()
+
+    assert _stage_of(task_id) == "development"
+    ci.assert_not_called()
+    spy.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# TC-04: per-stage grace — fresh task untouched, at-threshold task eligible.
+# ---------------------------------------------------------------------------
+def test_tc04_grace_boundary(monkeypatch):
+    monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600)
+    _green_ci(monkeypatch)
+
+    fresh = _make_task("development", branch="feature/ET-002-fresh",
+                       wi="ET-002", age_s=10)        # < grace -> untouched
+    stuck = _make_task("development", branch="feature/ET-003-stuck",
+                       wi="ET-003", age_s=3600)       # >= grace -> advanced
+
+    Reconciler().reconcile_gate_once()
+
+    assert _stage_of(fresh) == "development"
+    assert _stage_of(stuck) == "review"
+
+
+# ---------------------------------------------------------------------------
+# TC-05: grace_for_stage reads overrides JSON; bad JSON -> default, no crash.
+# ---------------------------------------------------------------------------
+def test_tc05_grace_for_stage_overrides(monkeypatch):
+    monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600)
+    monkeypatch.setattr(
+        reconciler_mod.settings,
+        "reconcile_grace_overrides_json",
+        '{"development": 30, "review": 7200}',
+    )
+    assert grace_for_stage("development") == 30
+    assert grace_for_stage("review") == 7200
+    # missing key -> default
+    assert grace_for_stage("testing") == 600
+
+
+def test_tc05_grace_for_stage_invalid_json_falls_back(monkeypatch):
+    monkeypatch.setattr(reconciler_mod.settings, "reconcile_grace_default_s", 600)
+    monkeypatch.setattr(
+        reconciler_mod.settings, "reconcile_grace_overrides_json", "{not valid json"
+    )
+    # Must not raise, must fall back to the default.
+    assert grace_for_stage("development") == 600
+
+
+# ---------------------------------------------------------------------------
+# TC-06: no spam — a stable-red gate never advances and never notifies, even
+#        across many ticks.
+# ---------------------------------------------------------------------------
+def test_tc06_red_gate_no_spam(monkeypatch):
+    _green_ci(monkeypatch, value=(False, "CI red"))
+    task_id = _make_task("development", age_s=3600)
+
+    rec = Reconciler()
+    for _ in range(5):
+        rec.reconcile_gate_once()
+
+    assert _stage_of(task_id) == "development"
+    # The QG-failure notification branch inside advance_stage must never fire,
+    # because advance_if_gate_passed returns None on a red gate (no advance call).
+    stage_engine.notify_qg_failure.assert_not_called()
+    stage_engine.plane_notify_qg.assert_not_called()
+    assert rec.unblocked_total == 0
+
+
+# ---------------------------------------------------------------------------
+# TC-07: silence when in sync — done / busy / within-grace tasks => no advance.
+# ---------------------------------------------------------------------------
+def test_tc07_silence_when_in_sync(monkeypatch):
+    _green_ci(monkeypatch)
+    spy = MagicMock(wraps=stage_engine.advance_stage)
+    monkeypatch.setattr(stage_engine, "advance_stage", spy)
+
+    _make_task("done", branch="feature/ET-010-done", wi="ET-010", age_s=3600)
+    fresh = _make_task("development", branch="feature/ET-011-fresh",
+                       wi="ET-011", age_s=5)
+    busy = _make_task("development", branch="feature/ET-012-busy",
+                      wi="ET-012", age_s=3600)
+    enqueue_job("reviewer", "enduro-trails", task_id=busy)
+
+    rec = Reconciler()
+    rec.reconcile_gate_once()
+
+    spy.assert_not_called()
+    assert rec.unblocked_total == 0
+    assert _stage_of(fresh) == "development"
+
+
+# ---------------------------------------------------------------------------
+# TC-08 (AC-16): F-1 never advances the human analysis gate.
+# ---------------------------------------------------------------------------
+def test_tc08_analysis_not_advanced_by_f1(monkeypatch):
+    # Even if the analysis gate would "pass", F-1 must not touch analysis.
+    monkeypatch.setitem(
+        stage_engine.QG_CHECKS, "check_analysis_approved",
+        MagicMock(return_value=(True, "approved")),
+    )
+    spy = MagicMock(wraps=stage_engine.advance_stage)
+    monkeypatch.setattr(stage_engine, "advance_stage", spy)
+
+    task_id = _make_task("analysis", age_s=3600)
+    Reconciler().reconcile_gate_once()
+
+    assert _stage_of(task_id) == "analysis"
+    spy.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# TC-09: never-raise — one task blowing up does not stop the others.
+# ---------------------------------------------------------------------------
+def test_tc09_never_raise_isolates_failure(monkeypatch):
+    calls = []
+
+    def boom(task_id, stage, repo, wi, branch):
+        calls.append(task_id)
+        raise RuntimeError("boom")
+
+    monkeypatch.setattr(reconciler_mod, "advance_if_gate_passed", boom)
+
+    t1 = _make_task("development", branch="feature/ET-020-a", wi="ET-020", age_s=3600)
+    t2 = _make_task("development", branch="feature/ET-021-b", wi="ET-021", age_s=3600)
+
+    # Must not raise despite both tasks raising inside advance_if_gate_passed.
+    Reconciler().reconcile_gate_once()
+
+    assert set(calls) == {t1, t2}  # both attempted
+
+
+# ---------------------------------------------------------------------------
+# TC-10: kill-switches.
+# ---------------------------------------------------------------------------
+def test_tc10_kill_switch_disables_gate(monkeypatch):
+    monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", False)
+    spy = MagicMock(wraps=stage_engine.advance_stage)
+    monkeypatch.setattr(stage_engine, "advance_stage", spy)
+    _green_ci(monkeypatch)
+
+    task_id = _make_task("development", age_s=3600)
+    Reconciler().reconcile_gate_once()
+
+    assert _stage_of(task_id) == "development"
+    spy.assert_not_called()
+
+
+def test_tc10_plane_switch_mutes_only_f2(monkeypatch):
+    monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", True)
+    monkeypatch.setattr(reconciler_mod.settings, "reconcile_plane_enabled", False)
+
+    plane_pass = MagicMock()
+    monkeypatch.setattr(reconciler_mod.Reconciler, "_reconcile_plane_project", plane_pass)
+    # F-2 muted -> reconcile_plane_once is a no-op.
+    Reconciler().reconcile_plane_once()
+    plane_pass.assert_not_called()
+
+    # F-1 still runs.
+    _green_ci(monkeypatch)
+    task_id = _make_task("development", age_s=3600)
+    Reconciler().reconcile_gate_once()
+    assert _stage_of(task_id) == "review"
+
+
+# ---------------------------------------------------------------------------
+# TC-20: observability — explicit unblock log line + telegram (AC-12 / F-4).
+# ---------------------------------------------------------------------------
+def test_tc20_unblock_logs_and_notifies(monkeypatch, caplog):
+    _green_ci(monkeypatch)
+    monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", True)
+    tg = MagicMock()
+    monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
+
+    _make_task("development", wi="ET-042", age_s=3600)
+
+    rec = Reconciler()
+    with caplog.at_level("INFO", logger="orchestrator.reconciler"):
+        rec.reconcile_gate_once()
+
+    # Exact AC-12 contract string.
+    assert "reconciler: ET-042 development разблокирована (потерян webhook)" in caplog.text
+    assert rec.unblocked_total == 1
+    assert rec.last_unblocked == "ET-042"
+    tg.assert_called_once()
+
+
+def test_tc20_no_telegram_when_disabled(monkeypatch):
+    _green_ci(monkeypatch)
+    monkeypatch.setattr(reconciler_mod.settings, "reconcile_notify_unblock", False)
+    tg = MagicMock()
+    monkeypatch.setattr(reconciler_mod, "send_telegram", tg)
+
+    _make_task("development", wi="ET-043", age_s=3600)
+    Reconciler().reconcile_gate_once()
+
+    tg.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# TC-21: restart-safe daemon thread — start/stop/idempotent start.
+# ---------------------------------------------------------------------------
+def test_tc21_daemon_thread_lifecycle(monkeypatch):
+    # Avoid any real work in the loop: disable both branches, big interval.
+    monkeypatch.setattr(reconciler_mod.settings, "reconcile_enabled", False)
+    rec = Reconciler(interval_s=60)
+
+    rec.start()
+    assert rec._thread is not None and rec._thread.is_alive()
+    first_thread = rec._thread
+
+    # Idempotent: a second start does not spawn a new thread.
+    rec.start()
+    assert rec._thread is first_thread
+
+    rec.stop(timeout=5.0)
+    assert not first_thread.is_alive()

tests/test_reconciler_plane.py

@@ -0,0 +1,297 @@
+"""ORCH-053: tests for the Plane-side reconciler (F-2) + sha-resolve helpers.
+
+F-2 polls the Plane API per project (``list_issues_by_state``) and REPLAYS a
+missed In Progress / Approved / Rejected transition through the EXISTING
+``webhooks.plane.handle_status_start`` / ``handle_verdict`` handlers — it never
+duplicates pipeline logic. These tests mock those handlers (AsyncMock) and the
+Plane API helpers, and verify the dispatch / idempotency / multi-project rules.
+
+TC-15 is the AC-4 anti-dup integration test for ``create_task_atomic`` against a
+real isolated sqlite DB under concurrency.
+TC-16 exercises ``plane_sync.list_issues_by_state`` directly (pagination + the
+never-raise contract).
+"""
+
+import os
+import tempfile
+import threading
+from types import SimpleNamespace
+
+import pytest
+
+_test_db = os.path.join(tempfile.gettempdir(), "test_orchestrator_reconciler_plane.db")
+os.environ["ORCH_DB_PATH"] = _test_db
+os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
+os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
+os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
+
+from unittest.mock import AsyncMock, MagicMock  # noqa: E402
+
+import src.db as _db  # noqa: E402
+from src.db import init_db, get_db, enqueue_job, create_task_atomic  # noqa: E402
+from src import reconciler as reconciler_mod  # noqa: E402
+from src import plane_sync  # noqa: E402
+from src.reconciler import Reconciler  # noqa: E402
+
+_IN_PROGRESS = "uuid-in-progress"
+_APPROVED = "uuid-approved"
+_REJECTED = "uuid-rejected"
+_OLD_TS = "2020-01-01T00:00:00Z"  # well past any grace
+
+
+@pytest.fixture(autouse=True)
+def fresh_db(monkeypatch):
+    monkeypatch.setattr(_db.settings, "db_path", _test_db)
+    if os.path.exists(_test_db):
+        os.unlink(_test_db)
+    init_db()
+    yield
+
+
+@pytest.fixture
+def single_project(monkeypatch):
+    """Restrict F-2 to a single fake project and stub its state resolution."""
+    proj = SimpleNamespace(
+        plane_project_id="proj-1", repo="enduro-trails", work_item_prefix="ET",
+    )
+    monkeypatch.setattr(reconciler_mod.projects, "PROJECTS", [proj])
+    monkeypatch.setattr(
+        reconciler_mod, "get_project_states",
+        lambda pid: {
+            "in_progress": _IN_PROGRESS,
+            "approved": _APPROVED,
+            "rejected": _REJECTED,
+        },
+    )
+    return proj
+
+
+def _make_task(plane_id, stage="review", repo="enduro-trails",
+               branch="feature/ET-001-x", wi="ET-001"):
+    conn = get_db()
+    cur = conn.execute(
+        "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage, plane_issue_id) "
+        "VALUES (?, ?, ?, ?, ?, ?)",
+        (plane_id, wi, repo, branch, stage, plane_id),
+    )
+    tid = cur.lastrowid
+    conn.commit()
+    conn.close()
+    return tid
+
+
+def _patch_handlers(monkeypatch):
+    start = AsyncMock()
+    verdict = AsyncMock()
+    monkeypatch.setattr(reconciler_mod, "handle_status_start", start)
+    monkeypatch.setattr(reconciler_mod, "handle_verdict", verdict)
+    return start, verdict
+
+
+def _patch_issues(monkeypatch, issues):
+    monkeypatch.setattr(
+        reconciler_mod, "list_issues_by_state", lambda pid, states: list(issues)
+    )
+
+
+# ---------------------------------------------------------------------------
+# TC-11: In Progress without a task -> handle_status_start once.
+# ---------------------------------------------------------------------------
+def test_tc11_in_progress_without_task_starts_pipeline(monkeypatch, single_project):
+    start, verdict = _patch_handlers(monkeypatch)
+    _patch_issues(monkeypatch, [
+        {"id": "iss-1", "state": {"id": _IN_PROGRESS}, "updated_at": _OLD_TS,
+         "name": "Some issue"},
+    ])
+
+    Reconciler().reconcile_plane_once()
+
+    assert start.call_count == 1
+    issue_data, project_id = start.call_args.args
+    assert issue_data["id"] == "iss-1"
+    assert issue_data["state"]["id"] == _IN_PROGRESS
+    assert project_id == "proj-1"
+    verdict.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# TC-12: Approved with an existing task, no active job -> handle_verdict(True).
+# ---------------------------------------------------------------------------
+def test_tc12_approved_replays_verdict(monkeypatch, single_project):
+    start, verdict = _patch_handlers(monkeypatch)
+    _make_task("iss-2", stage="review")
+    _patch_issues(monkeypatch, [
+        {"id": "iss-2", "state": {"id": _APPROVED}, "updated_at": _OLD_TS},
+    ])
+
+    Reconciler().reconcile_plane_once()
+
+    assert verdict.call_count == 1
+    assert verdict.call_args.kwargs.get("approved") is True
+    start.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# TC-13: Rejected with an existing task -> handle_verdict(False).
+# ---------------------------------------------------------------------------
+def test_tc13_rejected_replays_verdict(monkeypatch, single_project):
+    start, verdict = _patch_handlers(monkeypatch)
+    _make_task("iss-3", stage="review")
+    _patch_issues(monkeypatch, [
+        {"id": "iss-3", "state": {"id": _REJECTED}, "updated_at": _OLD_TS},
+    ])
+
+    Reconciler().reconcile_plane_once()
+
+    assert verdict.call_count == 1
+    assert verdict.call_args.kwargs.get("approved") is False
+    start.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# TC-14: idempotency — an active job means a live webhook is in flight -> skip.
+# ---------------------------------------------------------------------------
+def test_tc14_active_job_skips(monkeypatch, single_project):
+    start, verdict = _patch_handlers(monkeypatch)
+    tid = _make_task("iss-4", stage="review")
+    enqueue_job("reviewer", "enduro-trails", task_id=tid)  # active
+    _patch_issues(monkeypatch, [
+        {"id": "iss-4", "state": {"id": _APPROVED}, "updated_at": _OLD_TS},
+    ])
+
+    Reconciler().reconcile_plane_once()
+
+    start.assert_not_called()
+    verdict.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# TC-14b: within-grace issue is left alone (lost, not merely delayed).
+# ---------------------------------------------------------------------------
+def test_tc14b_within_grace_skipped(monkeypatch, single_project):
+    from datetime import datetime, timezone
+    start, verdict = _patch_handlers(monkeypatch)
+    _make_task("iss-5", stage="review")
+    fresh_ts = datetime.now(timezone.utc).isoformat()
+    _patch_issues(monkeypatch, [
+        {"id": "iss-5", "state": {"id": _APPROVED}, "updated_at": fresh_ts},
+    ])
+
+    Reconciler().reconcile_plane_once()
+
+    start.assert_not_called()
+    verdict.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# TC-15 (AC-4): atomic anti-dup — concurrent create_task_atomic for one
+#               plane_id yields exactly ONE row and ONE created=True.
+# ---------------------------------------------------------------------------
+def test_tc15_create_task_atomic_no_duplicate():
+    results = []
+    barrier = threading.Barrier(8)
+
+    def worker():
+        barrier.wait()  # maximise the race
+        row, created = create_task_atomic(
+            "plane-dup", "ET-099", "enduro-trails",
+            "feature/ET-099-x", "analysis", "Dup race",
+        )
+        results.append((row["id"], created))
+
+    threads = [threading.Thread(target=worker) for _ in range(8)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+
+    created_flags = [c for _, c in results]
+    assert created_flags.count(True) == 1   # exactly one winner
+    assert created_flags.count(False) == 7  # the rest see the existing row
+
+    conn = get_db()
+    n = conn.execute(
+        "SELECT COUNT(*) FROM tasks WHERE plane_id = 'plane-dup'"
+    ).fetchone()[0]
+    conn.close()
+    assert n == 1  # only one task row ever created
+
+    # All callers see the same row id (the single task).
+    assert len({rid for rid, _ in results}) == 1
+
+
+# ---------------------------------------------------------------------------
+# TC-16: list_issues_by_state — never-raise on API error, filter+paginate on OK.
+# ---------------------------------------------------------------------------
+def test_tc16_list_issues_never_raises_on_error(monkeypatch):
+    def boom(*a, **k):
+        raise RuntimeError("plane down")
+
+    monkeypatch.setattr(plane_sync.httpx, "get", boom)
+    out = plane_sync.list_issues_by_state("proj-1", [_APPROVED])
+    assert out == []
+
+
+def test_tc16_list_issues_paginates_and_filters(monkeypatch):
+    page1 = {
+        "results": [
+            {"id": "a", "state": {"id": _APPROVED}},
+            {"id": "b", "state": {"id": "other"}},
+        ],
+        "next_page_results": True,
+        "next_cursor": "cur2",
+    }
+    page2 = {
+        "results": [
+            {"id": "c", "state": _APPROVED},          # bare-uuid state shape
+            {"id": "d", "state": {"id": _REJECTED}},
+        ],
+        "next_page_results": False,
+        "next_cursor": None,
+    }
+    pages = iter([page1, page2])
+
+    def fake_get(url, headers=None, params=None, timeout=None):
+        resp = MagicMock()
+        resp.json.return_value = next(pages)
+        resp.raise_for_status.return_value = None
+        return resp
+
+    monkeypatch.setattr(plane_sync.httpx, "get", fake_get)
+
+    out = plane_sync.list_issues_by_state("proj-1", [_APPROVED, _REJECTED])
+    ids = {i["id"] for i in out}
+    assert ids == {"a", "c", "d"}  # 'b' filtered out (state 'other')
+
+
+# ---------------------------------------------------------------------------
+# TC-17: F-2 polls EVERY registry project and resolves states per-project.
+# ---------------------------------------------------------------------------
+def test_tc17_polls_all_projects_resolves_states_per_project(monkeypatch):
+    _patch_handlers(monkeypatch)
+    from src import projects as projects_mod
+    projects_mod.reload_projects()
+    expected_ids = {p.plane_project_id for p in projects_mod.PROJECTS}
+    assert len(expected_ids) >= 2  # enduro + orchestrator in the default registry
+
+    states_calls = []
+    issues_calls = []
+
+    def fake_states(pid):
+        states_calls.append(pid)
+        return {"in_progress": _IN_PROGRESS, "approved": _APPROVED, "rejected": _REJECTED}
+
+    def fake_issues(pid, states):
+        issues_calls.append((pid, tuple(states)))
+        return []
+
+    monkeypatch.setattr(reconciler_mod, "get_project_states", fake_states)
+    monkeypatch.setattr(reconciler_mod, "list_issues_by_state", fake_issues)
+
+    Reconciler().reconcile_plane_once()
+
+    assert set(states_calls) == expected_ids
+    assert {pid for pid, _ in issues_calls} == expected_ids
+    # state uuids are resolved per-project (not hardcoded): each call carries them.
+    for _pid, states in issues_calls:
+        assert set(states) == {_IN_PROGRESS, _APPROVED, _REJECTED}