fix(staging): tolerate sandbox-infra-only FAILs (C9a/C9b) in deploy-staging verdict

The self-hosting orchestrator looped on deploy-staging -> development because
scripts/staging_check.py exited 1 on ANY failed check, so two infra-only checks
(C9a sandbox branch / C9b analyst-job — caused by SANDBOX bot accounts not being
members of the sandbox Plane project, NOT a pipeline regress) forced
staging_status: FAILED -> rollback -> loop, burning developer retries and tokens.

Direction (б) per ADR-001: classify staging checks as REAL (all pipeline checks,
fail-closed) vs SANDBOX_INFRA (narrow allowlist {C9a, C9b}, waivable). New leaf
module src/staging_verdict.py (stdlib-only, never-raise): classify_check +
compute_staging_verdict fold per-check results into a tolerant-but-fail-closed
verdict — any REAL failure -> FAILED/exit1 (safety net holds under any flag);
only C9a/C9b failed & tolerant -> SUCCESS/exit0 with waived list; only infra &
strict -> FAILED/exit1; any internal error -> FAILED/exit1 (never a false green).

staging_check.py now auto-classifies each check (public 3-tuple _items shape kept
as an ORCH-048 b6 regression guard), exposes categorized_items(), prints
INFRA-WAIVED/VERDICT lines, and exits via the verdict; new --strict flag forces
legacy strictness per-run. Kill-switch ORCH_STAGING_INFRA_TOLERANCE_ENABLED
(default true) restores legacy strict mode globally. launcher gains
action_stage_no_changes_note so "no changes to commit" on action stages is logged
as expected, not treated as under-delivery.

Contracts unchanged: STAGE_TRANSITIONS, QG_CHECKS registry, staging_status:/
deploy_status: frontmatter, hook exit-code (0/1/2), check_staging_status; no DB
migration. Docs: README, STAGING_CHECK.md, deployer.md, .env.example, CHANGELOG.

Refs: ORCH-061

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

tests/test_config.py

@@ -142,3 +142,26 @@ def test_image_freshness_settings_env_override(monkeypatch):
     s = Settings()
     assert s.image_freshness_enabled is False
     assert s.image_freshness_repos == "orchestrator,enduro-trails"
+
+
+# ---------------------------------------------------------------------------
+# ORCH-061 / TC-09: staging_infra_tolerance_enabled kill-switch (AC-7).
+# ---------------------------------------------------------------------------
+def test_staging_infra_tolerance_defaults_true(monkeypatch):
+    """TC-09 / AC-7: the kill-switch defaults ON (safe default — the safety net
+    holds regardless; the flag exists to restore legacy strictness instantly)."""
+    monkeypatch.delenv("ORCH_STAGING_INFRA_TOLERANCE_ENABLED", raising=False)
+    assert Settings().staging_infra_tolerance_enabled is True
+
+
+def test_staging_infra_tolerance_env_override_false(monkeypatch):
+    """TC-09 / AC-7: ORCH_STAGING_INFRA_TOLERANCE_ENABLED=false -> strict (1:1
+    pre-ORCH-061: infra-only FAIL again rolls back)."""
+    monkeypatch.setenv("ORCH_STAGING_INFRA_TOLERANCE_ENABLED", "false")
+    assert Settings().staging_infra_tolerance_enabled is False
+
+
+def test_staging_infra_tolerance_env_override_true(monkeypatch):
+    """The field is read verbatim from its ORCH_* env var."""
+    monkeypatch.setenv("ORCH_STAGING_INFRA_TOLERANCE_ENABLED", "true")
+    assert Settings().staging_infra_tolerance_enabled is True

tests/test_launcher.py

@@ -278,3 +278,48 @@ class TestWatchdogGracefulKill:
 
         assert signal.SIGKILL not in sent
         assert recorded["called"] is False
+
+
+# ---------------------------------------------------------------------------
+# ORCH-061 / TC-06 + TC-07: "no changes to commit" on an action stage is EXPECTED,
+# not under-delivery (FR-3 / AC-4). action_stage_no_changes_note is the PURE
+# observability decision used by the post-run no-changes branch: it returns an
+# explicit note for self-deploy action stages (deploy-staging/deploy) and None
+# everywhere else. It NEVER signals a rollback — advancement is decided by the
+# exit-code + gate verdict, never by a commit existing.
+# ---------------------------------------------------------------------------
+from src.agents.launcher import action_stage_no_changes_note  # noqa: E402
+
+
+class TestActionStageNoChangesNote:
+    def test_tc06_deploy_staging_self_deploy_returns_note(self):
+        """TC-06 / AC-4: on deploy-staging for the self-hosting repo, an empty diff
+        yields an explicit "expected on action stage" note (no rollback signal)."""
+        note = action_stage_no_changes_note("deploy-staging", "orchestrator")
+        assert note is not None
+        assert "deploy-staging" in note
+        assert "expected on action stage" in note
+
+    def test_tc06_deploy_self_deploy_returns_note(self):
+        """The `deploy` stage is equally an action stage for self-deploy."""
+        note = action_stage_no_changes_note("deploy", "orchestrator")
+        assert note is not None
+        assert "deploy: no code changes" in note
+
+    def test_tc07_development_stage_returns_none(self):
+        """TC-07 / AC-4 regression-guard: on a CODE stage (development) the new
+        action-stage allowance does NOT apply — no note, behaviour unchanged."""
+        assert action_stage_no_changes_note("development", "orchestrator") is None
+
+    def test_tc06_non_self_repo_returns_none(self):
+        """Conditionality (FR-5): the action-stage allowance is self-deploy only;
+        a non-self repo on deploy-staging gets no special note."""
+        assert action_stage_no_changes_note("deploy-staging", "enduro-trails") is None
+
+    def test_review_stage_returns_none(self):
+        """Any non-action stage -> None (defensive: only deploy stages qualify)."""
+        assert action_stage_no_changes_note("review", "orchestrator") is None
+
+    def test_never_raises_on_bad_input(self):
+        """never-raise: odd inputs (None stage / None repo) degrade to None."""
+        assert action_stage_no_changes_note(None, None) is None

tests/test_qg.py

@@ -689,6 +689,27 @@ class TestCheckStagingStatus:
         assert passed is True
         assert "N/A" in reason
 
+    # ------------------------------------------------------------------
+    # ORCH-061 / TC-08: the conditional staging gate is unchanged for
+    # non-self-hosting repos AND independent of the new tolerance flag (FR-5/AC-6).
+    # ------------------------------------------------------------------
+
+    def test_tc08_non_self_na_independent_of_tolerance_flag(self, tmp_path, monkeypatch):
+        """TC-08 / AC-6: for a non-self-hosting repo check_staging_status is the
+        byte-identical (True, "Staging gate N/A …") regardless of whether the
+        ORCH-061 infra-tolerance flag is on or off — the new behaviour never
+        activates off the self-hosting path."""
+        monkeypatch.setattr("src.qg.checks.settings.repos_dir", str(tmp_path))
+        from src.qg.checks import check_staging_status
+        for flag in (True, False):
+            monkeypatch.setattr(
+                "src.config.settings.staging_infra_tolerance_enabled", flag,
+                raising=False,
+            )
+            passed, reason = check_staging_status("enduro-trails", "ET-035")
+            assert passed is True
+            assert reason == "Staging gate N/A for enduro-trails"
+
     # ------------------------------------------------------------------
     # is_self_hosting_repo helper
     # ------------------------------------------------------------------

tests/test_qg_checks.py

@@ -51,3 +51,100 @@ def test_tc15_finalizer_log_roundtrips_through_parser():
     ok_f, _ = _parse_deploy_status(build_deploy_log("ORCH-036", 2, "FAILED"))
     assert ok_s is True
     assert ok_f is False
+
+
+# ---------------------------------------------------------------------------
+# ORCH-061 / TC-04 + TC-05: infra-tolerant staging verdict (pure logic, AC-2/AC-3).
+#
+# compute_staging_verdict folds the staging-check suite into a single
+# SUCCESS/FAILED verdict that is TOLERANT to known sandbox-infra failures
+# (C9a/C9b) but stays fail-closed for any REAL pipeline check. These tests
+# exercise the verdict directly — no live staging stand / docker (02-trz §9).
+# ---------------------------------------------------------------------------
+from src.staging_verdict import (  # noqa: E402
+    REAL,
+    SANDBOX_INFRA,
+    compute_staging_verdict,
+)
+
+
+def _rows(*specs):
+    """Helper: build (label, passed, category) rows."""
+    return [(label, passed, cat) for label, passed, cat in specs]
+
+
+def test_tc04_only_infra_failures_waived_to_success():
+    """TC-04 / AC-2: every REAL check PASS, only known sandbox-infra checks
+    (C9a/C9b) FAIL, tolerance ON -> SUCCESS / exit 0 (no false rollback)."""
+    rows = _rows(
+        ("C7 Create issue in Plane SANDBOX", True, REAL),
+        ("C8 Trigger pipeline via /webhook/plane", True, REAL),
+        ("C9a Branch appears in orchestrator-sandbox", False, SANDBOX_INFRA),
+        ("C9b Analyst job enqueued in staging queue", False, SANDBOX_INFRA),
+    )
+    v = compute_staging_verdict(rows, infra_tolerant=True)
+    assert v.status == "SUCCESS"
+    assert v.exit_code == 0
+    # Both infra checks are surfaced as waived (observability, FR-7).
+    assert set(v.waived) == {
+        "C9a Branch appears in orchestrator-sandbox",
+        "C9b Analyst job enqueued in staging queue",
+    }
+
+
+def test_tc05_any_real_failure_fails_closed():
+    """TC-05 / AC-3: at least one REAL pipeline check FAILS (alongside the infra
+    ones) -> FAILED / exit 1 even with tolerance ON (safety net not weakened)."""
+    rows = _rows(
+        ("C7 Create issue in Plane SANDBOX", False, REAL),   # real regression
+        ("C8 Trigger pipeline via /webhook/plane", True, REAL),
+        ("C9a Branch appears in orchestrator-sandbox", False, SANDBOX_INFRA),
+    )
+    v = compute_staging_verdict(rows, infra_tolerant=True)
+    assert v.status == "FAILED"
+    assert v.exit_code == 1
+    assert v.waived == []   # nothing waived when a real check failed
+
+
+def test_tc05_real_failure_fails_closed_even_alone():
+    """A single REAL failure (no infra failures) is still FAILED (fail-closed)."""
+    rows = _rows(("C7 Create issue in Plane SANDBOX", False, REAL))
+    v = compute_staging_verdict(rows, infra_tolerant=True)
+    assert v.status == "FAILED"
+    assert v.exit_code == 1
+
+
+def test_tc09_infra_failure_strict_mode_fails_closed():
+    """TC-09 / AC-7: with tolerance OFF, an infra-only FAIL again -> FAILED
+    (1:1 pre-ORCH-061 strict behaviour)."""
+    rows = _rows(
+        ("C7 Create issue in Plane SANDBOX", True, REAL),
+        ("C9a Branch appears in orchestrator-sandbox", False, SANDBOX_INFRA),
+    )
+    v = compute_staging_verdict(rows, infra_tolerant=False)
+    assert v.status == "FAILED"
+    assert v.exit_code == 1
+
+
+def test_all_green_is_success_regardless_of_tolerance():
+    rows = _rows(
+        ("C7 Create issue in Plane SANDBOX", True, REAL),
+        ("C9a Branch appears in orchestrator-sandbox", True, SANDBOX_INFRA),
+    )
+    for tol in (True, False):
+        v = compute_staging_verdict(rows, infra_tolerant=tol)
+        assert v.status == "SUCCESS"
+        assert v.exit_code == 0
+        assert v.waived == []
+
+
+def test_tc12_compute_verdict_never_raises_on_garbage():
+    """AC-10 never-raise: malformed rows degrade to a conservative FAILED, never
+    an exception."""
+    v = compute_staging_verdict([("only-one-element",)], infra_tolerant=True)
+    assert v.status == "FAILED"
+    assert v.exit_code == 1
+    # A completely broken iterable also fails closed without raising.
+    v2 = compute_staging_verdict(None, infra_tolerant=True)
+    assert v2.status == "FAILED"
+    assert v2.exit_code == 1

tests/test_stage_engine.py

@@ -1132,3 +1132,158 @@ class TestDelegation:
         assert args[0] == 5
         assert args[1] == "analysis"
         assert args[-1] is None
+
+
+# ---------------------------------------------------------------------------
+# ORCH-061: no deploy-staging loop on a healthy self-deploy; the ORCH-35 safety
+# net (real staging FAIL -> rollback) stays intact; the new logic never raises
+# into advance_stage; and "green with an infra allowance" is distinguishable from
+# an honest green (observability).
+# ---------------------------------------------------------------------------
+class TestStagingInfraTolerance:
+    """The verdict that produces ``staging_status:`` is computed in the suite
+    BEFORE the gate (ORCH-061 ADR-001 §4: check_staging_status is unchanged). At
+    the engine level we therefore assert the REACTION to the resulting verdict:
+    SUCCESS advances (no loop), a REAL FAILED rolls back (safety net)."""
+
+    def _patch_self_deploy_state(self, monkeypatch, tmp_path):
+        # Phase A writes restart-safe markers under repos_dir — keep them in tmp.
+        monkeypatch.setattr(stage_engine.self_deploy.settings, "repos_dir", str(tmp_path))
+        monkeypatch.setattr(stage_engine.self_deploy.settings, "host_repos_dir", str(tmp_path))
+
+    def test_tc01_healthy_self_deploy_advances_no_rollback(self, monkeypatch, tmp_path):
+        """TC-01 / AC-1: staging SUCCESS (infra-FAIL already waived in the suite)
+        + green merge/freshness sub-gates -> deploy-staging advances to `deploy`
+        (Phase A approval-pending). NO rollback to development (loop is gone)."""
+        self._patch_self_deploy_state(monkeypatch, tmp_path)
+        monkeypatch.setattr(
+            stage_engine, "QG_CHECKS",
+            {**stage_engine.QG_CHECKS,
+             "check_staging_status": _pass,
+             "check_branch_mergeable": _pass,
+             "check_staging_image_fresh": _pass},
+        )
+        task_id = _make_task("deploy-staging", repo="orchestrator", wi="ORCH-061",
+                             branch="feature/ORCH-061-x")
+        res = advance_stage(
+            task_id, "deploy-staging", "orchestrator", "ORCH-061",
+            "feature/ORCH-061-x", finished_agent="deployer",
+        )
+        assert res.advanced is True
+        assert res.to_stage == "deploy"
+        assert _stage(task_id) == "deploy"            # Phase A advanced the stage
+        assert res.rolled_back_to is None             # NO loop back to development
+        assert res.note == "self-deploy-approval-pending"
+
+    def test_tc02_real_staging_failed_rolls_back(self, monkeypatch, tmp_path):
+        """TC-02 / AC-3: a REAL staging failure (verdict FAILED) still rolls
+        deploy-staging back to development + set_issue_blocked + alert — the
+        ORCH-35 safety net is NOT weakened by the infra tolerance (FR-4)."""
+        self._patch_self_deploy_state(monkeypatch, tmp_path)
+        monkeypatch.setattr(
+            stage_engine, "QG_CHECKS",
+            {**stage_engine.QG_CHECKS,
+             "check_staging_status": _fail("Staging status: FAILED")},
+        )
+        task_id = _make_task("deploy-staging", repo="orchestrator", wi="ORCH-061",
+                             branch="feature/ORCH-061-x")
+        res = advance_stage(
+            task_id, "deploy-staging", "orchestrator", "ORCH-061",
+            "feature/ORCH-061-x", finished_agent="deployer",
+        )
+        assert res.advanced is False
+        assert res.rolled_back_to == "development"
+        assert _stage(task_id) == "development"
+        assert res.alerted is True
+        assert stage_engine.set_issue_blocked.called
+        assert stage_engine.send_telegram.called
+
+    def test_tc12_gate_exception_never_crashes_advance(self, monkeypatch, tmp_path):
+        """TC-12 / AC-10 never-raise: if the staging gate raises (io/parse/docker
+        hiccup), advance_stage catches it deterministically — no exception escapes,
+        the task does NOT advance and is NOT falsely rolled back to development."""
+        self._patch_self_deploy_state(monkeypatch, tmp_path)
+
+        def _boom(*a, **k):
+            raise RuntimeError("staging gate blew up")
+
+        monkeypatch.setattr(
+            stage_engine, "QG_CHECKS",
+            {**stage_engine.QG_CHECKS, "check_staging_status": _boom},
+        )
+        task_id = _make_task("deploy-staging", repo="orchestrator", wi="ORCH-061",
+                             branch="feature/ORCH-061-x")
+        # Must NOT raise.
+        res = advance_stage(
+            task_id, "deploy-staging", "orchestrator", "ORCH-061",
+            "feature/ORCH-061-x", finished_agent="deployer",
+        )
+        assert res.advanced is False
+        assert res.rolled_back_to is None             # exception != gate FAILED
+        assert _stage(task_id) == "deploy-staging"    # stays put, no loop
+        assert res.note and "error" in res.note
+
+    def test_tc13_end_to_end_self_deploy_no_single_rollback(self, monkeypatch, tmp_path):
+        """TC-13 / AC-1+AC-4 integration: a healthy self-deploy goes
+        deploy-staging -> deploy (Phase A) -> (approve/finalize SUCCESS) -> done
+        WITHOUT a single rollback to development in the transition log."""
+        self._patch_self_deploy_state(monkeypatch, tmp_path)
+        monkeypatch.setattr(
+            stage_engine, "QG_CHECKS",
+            {**stage_engine.QG_CHECKS,
+             "check_staging_status": _pass,
+             "check_branch_mergeable": _pass,
+             "check_staging_image_fresh": _pass,
+             "check_deploy_status": _pass},
+        )
+        task_id = _make_task("deploy-staging", repo="orchestrator", wi="ORCH-061",
+                             branch="feature/ORCH-061-x")
+
+        seen_stages = []
+
+        # 1) deploy-staging -> deploy (Phase A approval-pending).
+        r1 = advance_stage(
+            task_id, "deploy-staging", "orchestrator", "ORCH-061",
+            "feature/ORCH-061-x", finished_agent="deployer",
+        )
+        seen_stages.append(_stage(task_id))
+        assert r1.advanced is True
+        assert _stage(task_id) == "deploy"
+
+        # 2) finalizer (Phase C): deploy verdict SUCCESS -> done.
+        r2 = advance_stage(
+            task_id, "deploy", "orchestrator", "ORCH-061",
+            "feature/ORCH-061-x", finished_agent="deployer",
+        )
+        seen_stages.append(_stage(task_id))
+        assert r2.advanced is True
+        assert _stage(task_id) == "done"
+
+        # Not a single rollback to development anywhere in the path.
+        assert "development" not in seen_stages
+        assert r1.rolled_back_to is None and r2.rolled_back_to is None
+
+    def test_tc14_waived_green_distinguishable_from_honest_green(self):
+        """TC-14 / AC-11 observability: the staging verdict makes "green with an
+        infra allowance" distinguishable from an honest green — the waived list is
+        populated and the summary says so, vs an empty waived list + plain summary
+        for an all-green run."""
+        from src.staging_verdict import REAL, SANDBOX_INFRA, compute_staging_verdict
+
+        waived = compute_staging_verdict(
+            [("C7", True, REAL),
+             ("C9a", False, SANDBOX_INFRA)],
+            infra_tolerant=True,
+        )
+        honest = compute_staging_verdict(
+            [("C7", True, REAL),
+             ("C9a", True, SANDBOX_INFRA)],
+            infra_tolerant=True,
+        )
+        # Both advance...
+        assert waived.status == honest.status == "SUCCESS"
+        # ...but only the waived one carries the explicit allowance marker.
+        assert waived.waived == ["C9a"]
+        assert "infra-waived" in waived.summary.lower()
+        assert honest.waived == []
+        assert "infra-waived" not in honest.summary.lower()

tests/test_staging_check_b6.py

@@ -149,3 +149,70 @@ def test_run_b6_records_pass_for_clean_registry(monkeypatch):
     _label, passed, detail = results._items[0]
     assert passed is True
     assert "sandbox=YES" in detail
+
+
+# ---------------------------------------------------------------------------
+# ORCH-061 / TC-03: the suite classifies checks as REAL vs SANDBOX_INFRA so the
+# verdict (and exit-code) can tolerate KNOWN sandbox-infra FAILs (C9a/C9b) while
+# staying fail-closed for real pipeline checks. Tested without a live stand.
+# ---------------------------------------------------------------------------
+from src.staging_verdict import REAL, SANDBOX_INFRA  # noqa: E402
+
+
+def test_tc03_classify_infra_checks():
+    """C9a/C9b classify as SANDBOX_INFRA; pipeline checks (A/B/C7/C8) as REAL."""
+    assert sc._classify("C9a Branch appears in orchestrator-sandbox") == SANDBOX_INFRA
+    assert sc._classify("C9b Analyst job enqueued in staging queue") == SANDBOX_INFRA
+    assert sc._classify("C7 Create issue in Plane SANDBOX") == REAL
+    assert sc._classify("C8 Trigger pipeline via /webhook/plane") == REAL
+    assert sc._classify("A1 GET /health") == REAL
+    assert sc._classify("B6 Registry: sandbox present") == REAL
+
+
+def test_tc03_results_records_categories_and_keeps_tuple_shape():
+    """Results.add auto-classifies each check; categorized_items() exposes the
+    category WITHOUT changing the public 3-tuple shape of _items (ORCH-048 b6
+    tests still unpack (label, passed, detail))."""
+    results = sc.Results()
+    results.add("C7 Create issue in Plane SANDBOX", True)
+    results.add("C9a Branch appears in orchestrator-sandbox", False)
+
+    # Public _items shape unchanged (regression guard for ORCH-048 tests).
+    for item in results._items:
+        assert len(item) == 3
+
+    cats = {label: cat for label, _passed, cat in results.categorized_items()}
+    assert cats["C7 Create issue in Plane SANDBOX"] == REAL
+    assert cats["C9a Branch appears in orchestrator-sandbox"] == SANDBOX_INFRA
+
+
+def test_tc03_explicit_category_overrides_autoclassify():
+    """An explicit category arg is honoured (caller can force REAL)."""
+    results = sc.Results()
+    results.add("C9a Branch appears in orchestrator-sandbox", False, category=REAL)
+    label, _passed, cat = results.categorized_items()[0]
+    assert cat == REAL
+
+
+def test_tc03_suite_verdict_waives_infra_only_failure():
+    """End-to-end through the suite helpers: a run whose only failures are C9a/C9b
+    -> exit 0 (waived) under tolerance; the waiver is surfaced for observability."""
+    results = sc.Results()
+    results.add("C7 Create issue in Plane SANDBOX", True)
+    results.add("C8 Trigger pipeline via /webhook/plane", True)
+    results.add("C9a Branch appears in orchestrator-sandbox", False)
+    results.add("C9b Analyst job enqueued in staging queue", False)
+
+    verdict = sc._verdict(results.categorized_items(), infra_tolerant=True)
+    assert verdict.status == "SUCCESS"
+    assert verdict.exit_code == 0
+    assert len(verdict.waived) == 2
+
+    # Strict mode (kill-switch off) re-fails the same run.
+    strict = sc._verdict(results.categorized_items(), infra_tolerant=False)
+    assert strict.exit_code == 1
+
+
+def test_tc03_resolve_tolerance_strict_flag_forces_off():
+    """--strict forces tolerance OFF regardless of the config default."""
+    assert sc._resolve_tolerance(cli_strict=True) is False