A DB stage=done task with 0 active jobs flapped in Plane between `Awaiting Deploy` and `Monitoring after Deploy` instead of holding `Done` (verified live on ORCH-061, task 47): the three deploy-phase setters were terminal-blind, so any stale/duplicate/unknown caller under the bot token re-stamped an intermediate status over the terminal Done, forever. - New leaf src/deploy_status_guard.py (pure, never-raise, config-gated): decide() -> ALLOW | CONVERGE_DONE | SUPPRESS on the entry of set_issue_awaiting_deploy / set_issue_deploying / set_issue_monitoring. A deploy-phase status is legitimate iff the task is non-terminal OR (done AND post-deploy window active); otherwise done converges to Done idempotently, cancelled is suppressed (FR-2, D1/D2). - D3: move post_deploy.arm_monitor ABOVE the terminal-sync block in advance_stage so window_active is True when the legitimate first Monitoring is set (the task is already DB-done by then); a re-drive after the window closes converges to Done. - D4: run_post_deploy_monitor no-ops without a status PATCH / re-queue when the task became cancelled mid-window (zombie-tick guard, FR-3). - D5: additive `reason` kwarg on the three setters + one structured log line per verdict (work_item/caller/target/db_stage/window_active/verdict); new read-only db.get_task_by_work_item_id; post_deploy.window_active helper. - Flags deploy_status_guard_enabled (kill-switch -> 1:1) / deploy_status_guard_repos (CSV; empty = self-hosting only). STAGE_TRANSITIONS / QG_CHECKS / check_* / machine-verdict keys / DB schema untouched (reads existing tasks.stage). Tests: TC-01..TC-12 across 5 new test modules + config flags; updated the reason-kwarg assertions in test_deploy_terminal_sync / test_deploy_approve. Full regress green (1413). Docs: CHANGELOG, CLAUDE.md, docs/architecture/README.md (status -> реализовано), .env.example. Refs: ORCH-094 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
218 lines
9.1 KiB
Python
218 lines
9.1 KiB
Python
"""ORCH-094 — terminal-window-aware deploy-status guard (FR-2 / FR-5).
|
|
|
|
Covers (04-test-plan.yaml):
|
|
TC-01 deploy-status for a DB stage=done task converges to Done: a
|
|
set_issue_monitoring/awaiting/deploying attempt on a terminal task drives
|
|
Done (or no-op if already Done), never an intermediate status.
|
|
TC-02 idempotency: a repeated terminal-aware setter call on an already-Done task
|
|
never PATCHes an intermediate status (no Done<->deploy pendulum).
|
|
TC-03 a non-terminal task (stage=deploy) is NOT suppressed: the deploy setters
|
|
proceed normally (regression AC-4).
|
|
TC-04 kill-switch off -> 1:1 prior behaviour (guard inert); on -> converge.
|
|
TC-05 never-raise: an undeterminable DB stage / DB error degrades safely (ALLOW,
|
|
no flapp, no exception).
|
|
TC-12 non-self repo: zero regression — the guard is inert (self-hosting only).
|
|
"""
|
|
import os
|
|
import tempfile
|
|
|
|
import pytest
|
|
|
|
_test_db = os.path.join(tempfile.gettempdir(), "test_deploy_status_guard.db")
|
|
os.environ["ORCH_DB_PATH"] = _test_db
|
|
os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
|
|
os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
|
|
os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
|
|
|
|
from unittest.mock import MagicMock # noqa: E402
|
|
|
|
import src.db as _db # noqa: E402
|
|
from src.db import init_db, get_db # noqa: E402
|
|
from src import deploy_status_guard as guard # noqa: E402
|
|
from src import plane_sync # noqa: E402
|
|
from src import post_deploy # noqa: E402
|
|
from src import config as cfg # noqa: E402
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def fresh_db(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(_db.settings, "db_path", _test_db)
|
|
if os.path.exists(_test_db):
|
|
os.unlink(_test_db)
|
|
init_db()
|
|
# Guard ON, self-hosting only (empty CSV) by default.
|
|
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False)
|
|
monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False)
|
|
# post-deploy sentinels live under a fresh tmp dir (window closed by default).
|
|
monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path))
|
|
monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path))
|
|
yield
|
|
|
|
|
|
def _make_task(stage, repo="orchestrator", wi="ORCH-061", branch="feature/ORCH-061-x"):
|
|
conn = get_db()
|
|
conn.execute(
|
|
"INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
|
|
"VALUES (?, ?, ?, ?, ?)",
|
|
(f"plane-{wi}", wi, repo, branch, stage),
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def spy_setters(monkeypatch):
|
|
"""Spy the low-level PATCH primitive + the Done convergence target."""
|
|
direct = MagicMock()
|
|
done = MagicMock()
|
|
monkeypatch.setattr(plane_sync, "_set_issue_state_direct", direct)
|
|
monkeypatch.setattr(plane_sync, "set_issue_done", done)
|
|
# Keep status resolution offline-deterministic.
|
|
monkeypatch.setattr(plane_sync, "_resolve_project_id", lambda w=None, p=None: "proj-1")
|
|
monkeypatch.setattr(
|
|
plane_sync, "get_project_states",
|
|
lambda pid: {"awaiting_deploy": "S-aw", "deploying": "S-dep", "monitoring": "S-mon"},
|
|
)
|
|
return direct, done
|
|
|
|
|
|
# --- TC-01 ------------------------------------------------------------------
|
|
def test_tc01_done_task_converges_to_done(spy_setters):
|
|
direct, done = spy_setters
|
|
_make_task("done")
|
|
# Window is NOT active (no ARMED sentinel) -> Monitoring is spurious.
|
|
for setter in (
|
|
plane_sync.set_issue_monitoring,
|
|
plane_sync.set_issue_awaiting_deploy,
|
|
plane_sync.set_issue_deploying,
|
|
):
|
|
done.reset_mock()
|
|
direct.reset_mock()
|
|
setter("ORCH-061")
|
|
# Converged to Done; no intermediate deploy-status PATCH.
|
|
done.assert_called_once_with("ORCH-061")
|
|
direct.assert_not_called()
|
|
|
|
|
|
def test_tc01_decide_verdicts_for_done():
|
|
_make_task("done")
|
|
# No window -> all three converge.
|
|
assert guard.decide("ORCH-061", guard.MONITORING) == guard.CONVERGE_DONE
|
|
assert guard.decide("ORCH-061", guard.AWAITING) == guard.CONVERGE_DONE
|
|
assert guard.decide("ORCH-061", guard.DEPLOYING) == guard.CONVERGE_DONE
|
|
|
|
|
|
def test_tc01_decide_allows_monitoring_in_active_window(tmp_path, monkeypatch):
|
|
_make_task("done")
|
|
# Arm the window: ARMED present, DONE absent.
|
|
post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed")
|
|
assert post_deploy.window_active("orchestrator", "ORCH-061") is True
|
|
assert guard.decide("ORCH-061", guard.MONITORING) == guard.ALLOW
|
|
# Awaiting/Deploying are ALWAYS spurious for a done task, even with a window.
|
|
assert guard.decide("ORCH-061", guard.AWAITING) == guard.CONVERGE_DONE
|
|
# Once the window closes (DONE present) Monitoring converges too.
|
|
post_deploy.mark_done("orchestrator", "ORCH-061")
|
|
assert post_deploy.window_active("orchestrator", "ORCH-061") is False
|
|
assert guard.decide("ORCH-061", guard.MONITORING) == guard.CONVERGE_DONE
|
|
|
|
|
|
# --- TC-02 ------------------------------------------------------------------
|
|
def test_tc02_idempotent_no_pendulum(spy_setters):
|
|
direct, done = spy_setters
|
|
_make_task("done")
|
|
# Repeated calls keep converging to Done; the intermediate Monitoring PATCH
|
|
# never fires, so there is no Done<->deploy-status pendulum.
|
|
for _ in range(5):
|
|
plane_sync.set_issue_monitoring("ORCH-061")
|
|
assert direct.call_count == 0
|
|
assert done.call_count == 5 # idempotent PATCH-equivalent (same terminal state)
|
|
|
|
|
|
# --- TC-03 ------------------------------------------------------------------
|
|
def test_tc03_non_terminal_not_suppressed(spy_setters):
|
|
direct, done = spy_setters
|
|
_make_task("deploy") # a really-deploying task
|
|
plane_sync.set_issue_awaiting_deploy("ORCH-061")
|
|
plane_sync.set_issue_deploying("ORCH-061")
|
|
plane_sync.set_issue_monitoring("ORCH-061")
|
|
# All three proceed to a real PATCH; nothing converges to Done.
|
|
assert direct.call_count == 3
|
|
done.assert_not_called()
|
|
assert guard.decide("ORCH-061", guard.MONITORING) == guard.ALLOW
|
|
|
|
|
|
# --- TC-04 ------------------------------------------------------------------
|
|
def test_tc04_kill_switch(spy_setters, monkeypatch):
|
|
direct, done = spy_setters
|
|
_make_task("done")
|
|
# OFF -> terminal-blind, the monitoring PATCH proceeds (1:1 pre-ORCH-094).
|
|
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", False)
|
|
plane_sync.set_issue_monitoring("ORCH-061")
|
|
assert direct.call_count == 1
|
|
done.assert_not_called()
|
|
# ON -> converge to Done.
|
|
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True)
|
|
direct.reset_mock()
|
|
done.reset_mock()
|
|
plane_sync.set_issue_monitoring("ORCH-061")
|
|
direct.assert_not_called()
|
|
done.assert_called_once_with("ORCH-061")
|
|
|
|
|
|
# --- TC-05 ------------------------------------------------------------------
|
|
def test_tc05_never_raise_on_db_error(spy_setters, monkeypatch):
|
|
direct, done = spy_setters
|
|
_make_task("done")
|
|
|
|
def _boom(_wi):
|
|
raise RuntimeError("db down")
|
|
|
|
monkeypatch.setattr(_db, "get_task_by_work_item_id", _boom)
|
|
# decide degrades to ALLOW (fail-safe), never raises.
|
|
assert guard.decide("ORCH-061", guard.MONITORING) == guard.ALLOW
|
|
# The setter proceeds with the normal PATCH (1:1), no convergence, no crash.
|
|
plane_sync.set_issue_monitoring("ORCH-061")
|
|
assert direct.call_count == 1
|
|
done.assert_not_called()
|
|
|
|
|
|
def test_tc05_unknown_task_allows(spy_setters):
|
|
direct, done = spy_setters
|
|
# No task row at all -> ALLOW (foreign/unknown issue, not ours).
|
|
assert guard.decide("ORCH-999", guard.MONITORING) == guard.ALLOW
|
|
plane_sync.set_issue_monitoring("ORCH-999")
|
|
assert direct.call_count == 1
|
|
done.assert_not_called()
|
|
|
|
|
|
def test_tc05_cancelled_is_suppressed(spy_setters):
|
|
direct, done = spy_setters
|
|
_make_task("cancelled")
|
|
assert guard.decide("ORCH-061", guard.MONITORING) == guard.SUPPRESS
|
|
plane_sync.set_issue_monitoring("ORCH-061")
|
|
# Suppressed: neither an intermediate PATCH nor a Done convergence.
|
|
direct.assert_not_called()
|
|
done.assert_not_called()
|
|
|
|
|
|
# --- TC-12 ------------------------------------------------------------------
|
|
def test_tc12_non_self_repo_inert(spy_setters):
|
|
direct, done = spy_setters
|
|
# A non-self repo done task: the guard is inert (self-hosting only, empty CSV).
|
|
_make_task("done", repo="enduro-trails", wi="ET-042", branch="feature/ET-042-x")
|
|
assert guard.applies("enduro-trails") is False
|
|
assert guard.decide("ET-042", guard.MONITORING) == guard.ALLOW
|
|
plane_sync.set_issue_monitoring("ET-042")
|
|
# Behaviour unchanged: the requested PATCH proceeds, no convergence.
|
|
assert direct.call_count == 1
|
|
done.assert_not_called()
|
|
|
|
|
|
def test_tc12_csv_scope_overrides_self_hosting(monkeypatch):
|
|
_make_task("done", repo="enduro-trails", wi="ET-042", branch="feature/ET-042-x")
|
|
# Explicit CSV scope brings a non-self repo in-scope.
|
|
monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "enduro-trails")
|
|
assert guard.applies("enduro-trails") is True
|
|
assert guard.applies("orchestrator") is False # not listed -> out of scope
|
|
assert guard.decide("ET-042", guard.MONITORING) == guard.CONVERGE_DONE
|