fix(deploy): terminal-window-aware guard so done tasks hold Done in Plane (ORCH-094)

A DB stage=done task with 0 active jobs flapped in Plane between `Awaiting
Deploy` and `Monitoring after Deploy` instead of holding `Done` (verified live
on ORCH-061, task 47): the three deploy-phase setters were terminal-blind, so
any stale/duplicate/unknown caller under the bot token re-stamped an
intermediate status over the terminal Done, forever.

- New leaf src/deploy_status_guard.py (pure, never-raise, config-gated): decide()
  -> ALLOW | CONVERGE_DONE | SUPPRESS on the entry of set_issue_awaiting_deploy /
  set_issue_deploying / set_issue_monitoring. A deploy-phase status is legitimate
  iff the task is non-terminal OR (done AND post-deploy window active); otherwise
  done converges to Done idempotently, cancelled is suppressed (FR-2, D1/D2).
- D3: move post_deploy.arm_monitor ABOVE the terminal-sync block in advance_stage
  so window_active is True when the legitimate first Monitoring is set (the task
  is already DB-done by then); a re-drive after the window closes converges to Done.
- D4: run_post_deploy_monitor no-ops without a status PATCH / re-queue when the
  task became cancelled mid-window (zombie-tick guard, FR-3).
- D5: additive `reason` kwarg on the three setters + one structured log line per
  verdict (work_item/caller/target/db_stage/window_active/verdict); new read-only
  db.get_task_by_work_item_id; post_deploy.window_active helper.
- Flags deploy_status_guard_enabled (kill-switch -> 1:1) / deploy_status_guard_repos
  (CSV; empty = self-hosting only). STAGE_TRANSITIONS / QG_CHECKS / check_* /
  machine-verdict keys / DB schema untouched (reads existing tasks.stage).

Tests: TC-01..TC-12 across 5 new test modules + config flags; updated the
reason-kwarg assertions in test_deploy_terminal_sync / test_deploy_approve.
Full regress green (1413). Docs: CHANGELOG, CLAUDE.md, docs/architecture/README.md
(status -> реализовано), .env.example.

Refs: ORCH-094

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-09 23:31:30 +03:00
committed by orchestrator-deployer
parent db4dd275e4
commit a46dcbcab3
18 changed files with 1088 additions and 25 deletions

View File

@@ -292,3 +292,30 @@ def test_merge_retry_settings_env_override(monkeypatch):
assert s.merge_retry_max_attempts == 5
assert s.merge_retry_backoff_base_s == 1
assert s.merge_retry_backoff_max_s == 8
# ---------------------------------------------------------------------------
# ORCH-094: deploy_status_guard_* settings defaults + env override.
# ---------------------------------------------------------------------------
_DEPLOY_GUARD_ENV = (
"ORCH_DEPLOY_STATUS_GUARD_ENABLED",
"ORCH_DEPLOY_STATUS_GUARD_REPOS",
)
def test_deploy_status_guard_settings_defaults(monkeypatch):
"""Documented defaults: enabled True, repos empty (self-hosting only)."""
for name in _DEPLOY_GUARD_ENV:
monkeypatch.delenv(name, raising=False)
s = Settings()
assert s.deploy_status_guard_enabled is True
assert s.deploy_status_guard_repos == ""
def test_deploy_status_guard_settings_env_override(monkeypatch):
"""Each field is read from its ORCH_DEPLOY_STATUS_GUARD_* env var."""
monkeypatch.setenv("ORCH_DEPLOY_STATUS_GUARD_ENABLED", "false")
monkeypatch.setenv("ORCH_DEPLOY_STATUS_GUARD_REPOS", "orchestrator,enduro-trails")
s = Settings()
assert s.deploy_status_guard_enabled is False
assert s.deploy_status_guard_repos == "orchestrator,enduro-trails"

View File

@@ -132,7 +132,8 @@ def test_tc05_no_approve_does_not_call_prod_hook(monkeypatch):
# The restart-safe approve-requested marker was written.
assert self_deploy.has_marker("orchestrator", "ORCH-036", self_deploy.APPROVE_REQUESTED)
# ORCH-066 AC-6/AC-13: Phase A indicates `Awaiting Deploy`, NOT `In Review`.
stage_engine.set_issue_awaiting_deploy.assert_called_once_with("ORCH-036")
# ORCH-094: the caller now tags the reason (FR-4 observability).
stage_engine.set_issue_awaiting_deploy.assert_called_once_with("ORCH-036", reason="phase_a")
stage_engine.set_issue_in_review.assert_not_called()
@@ -161,7 +162,8 @@ def test_tc06_approved_calls_prod_hook_exactly_once(monkeypatch):
assert any(j["agent"] == "deploy-finalizer" for j in _jobs())
assert self_deploy.has_marker("orchestrator", "ORCH-036", self_deploy.INITIATED)
# ORCH-066 AC-7: Phase B indicates `Deploying` on a successful initiate.
stage_engine.set_issue_deploying.assert_called_once_with("ORCH-036")
# ORCH-094: the caller now tags the reason (FR-4 observability).
stage_engine.set_issue_deploying.assert_called_once_with("ORCH-036", reason="phase_b")
# 2nd (duplicate) Confirm Deploy -> idempotent no-op, hook NOT called again.
res2 = advance_stage(

View File

@@ -0,0 +1,88 @@
"""ORCH-094 — observability of deploy-status setting (FR-4 / AC-5 / TC-09).
Every deploy-phase status decision emits ONE structured line carrying work_item,
caller (reason), target_status, db_stage, window_active and the verdict; a
suppression/convergence is logged explicitly so a future flapp is attributable.
"""
import logging
import os
import tempfile
import pytest
_test_db = os.path.join(tempfile.gettempdir(), "test_deploy_status_obs.db")
os.environ["ORCH_DB_PATH"] = _test_db
os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
import src.db as _db # noqa: E402
from src.db import init_db, get_db # noqa: E402
from src import deploy_status_guard as guard # noqa: E402
from src import post_deploy # noqa: E402
from src import config as cfg # noqa: E402
@pytest.fixture(autouse=True)
def fresh_db(monkeypatch, tmp_path):
monkeypatch.setattr(_db.settings, "db_path", _test_db)
if os.path.exists(_test_db):
os.unlink(_test_db)
init_db()
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False)
monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False)
monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path))
monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path))
yield
def _make_task(stage, repo="orchestrator", wi="ORCH-061", branch="feature/ORCH-061-x"):
conn = get_db()
conn.execute(
"INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
"VALUES (?, ?, ?, ?, ?)",
(f"plane-{wi}", wi, repo, branch, stage),
)
conn.commit()
conn.close()
def test_tc09_converge_logs_full_attribution(caplog):
_make_task("done")
with caplog.at_level(logging.INFO, logger="orchestrator.deploy_status_guard"):
verdict = guard.decide("ORCH-061", guard.MONITORING, reason="advance:deploy->done")
assert verdict == guard.CONVERGE_DONE
rec = [r for r in caplog.records if r.name == "orchestrator.deploy_status_guard"]
assert rec, "guard emitted no observability record"
msg = rec[-1].getMessage()
# All five attribution fields + verdict are present.
for token in (
"work_item=ORCH-061", "caller=advance:deploy->done", "target=monitoring",
"db_stage=done", "window_active=False", "verdict=CONVERGE_DONE",
):
assert token in msg, f"missing {token!r} in {msg!r}"
# A convergence is logged at WARNING (easy to grep on a future flapp).
assert rec[-1].levelno == logging.WARNING
def test_tc09_allow_active_window_logged(caplog):
_make_task("done")
post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed")
with caplog.at_level(logging.INFO, logger="orchestrator.deploy_status_guard"):
verdict = guard.decide("ORCH-061", guard.MONITORING, reason="advance:deploy->done")
assert verdict == guard.ALLOW
rec = [r for r in caplog.records if r.name == "orchestrator.deploy_status_guard"][-1]
msg = rec.getMessage()
assert "window_active=True" in msg and "verdict=ALLOW" in msg
assert rec.levelno == logging.INFO
def test_tc09_suppress_cancelled_logged(caplog):
_make_task("cancelled")
with caplog.at_level(logging.INFO, logger="orchestrator.deploy_status_guard"):
verdict = guard.decide("ORCH-061", guard.AWAITING, reason="phase_a")
assert verdict == guard.SUPPRESS
rec = [r for r in caplog.records if r.name == "orchestrator.deploy_status_guard"][-1]
assert "verdict=SUPPRESS" in rec.getMessage()
assert "db_stage=cancelled" in rec.getMessage()
assert rec.levelno == logging.WARNING

View File

@@ -0,0 +1,217 @@
"""ORCH-094 — terminal-window-aware deploy-status guard (FR-2 / FR-5).
Covers (04-test-plan.yaml):
TC-01 deploy-status for a DB stage=done task converges to Done: a
set_issue_monitoring/awaiting/deploying attempt on a terminal task drives
Done (or no-op if already Done), never an intermediate status.
TC-02 idempotency: a repeated terminal-aware setter call on an already-Done task
never PATCHes an intermediate status (no Done<->deploy pendulum).
TC-03 a non-terminal task (stage=deploy) is NOT suppressed: the deploy setters
proceed normally (regression AC-4).
TC-04 kill-switch off -> 1:1 prior behaviour (guard inert); on -> converge.
TC-05 never-raise: an undeterminable DB stage / DB error degrades safely (ALLOW,
no flapp, no exception).
TC-12 non-self repo: zero regression — the guard is inert (self-hosting only).
"""
import os
import tempfile
import pytest
_test_db = os.path.join(tempfile.gettempdir(), "test_deploy_status_guard.db")
os.environ["ORCH_DB_PATH"] = _test_db
os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
from unittest.mock import MagicMock # noqa: E402
import src.db as _db # noqa: E402
from src.db import init_db, get_db # noqa: E402
from src import deploy_status_guard as guard # noqa: E402
from src import plane_sync # noqa: E402
from src import post_deploy # noqa: E402
from src import config as cfg # noqa: E402
@pytest.fixture(autouse=True)
def fresh_db(monkeypatch, tmp_path):
monkeypatch.setattr(_db.settings, "db_path", _test_db)
if os.path.exists(_test_db):
os.unlink(_test_db)
init_db()
# Guard ON, self-hosting only (empty CSV) by default.
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False)
monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False)
# post-deploy sentinels live under a fresh tmp dir (window closed by default).
monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path))
monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path))
yield
def _make_task(stage, repo="orchestrator", wi="ORCH-061", branch="feature/ORCH-061-x"):
conn = get_db()
conn.execute(
"INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
"VALUES (?, ?, ?, ?, ?)",
(f"plane-{wi}", wi, repo, branch, stage),
)
conn.commit()
conn.close()
@pytest.fixture
def spy_setters(monkeypatch):
"""Spy the low-level PATCH primitive + the Done convergence target."""
direct = MagicMock()
done = MagicMock()
monkeypatch.setattr(plane_sync, "_set_issue_state_direct", direct)
monkeypatch.setattr(plane_sync, "set_issue_done", done)
# Keep status resolution offline-deterministic.
monkeypatch.setattr(plane_sync, "_resolve_project_id", lambda w=None, p=None: "proj-1")
monkeypatch.setattr(
plane_sync, "get_project_states",
lambda pid: {"awaiting_deploy": "S-aw", "deploying": "S-dep", "monitoring": "S-mon"},
)
return direct, done
# --- TC-01 ------------------------------------------------------------------
def test_tc01_done_task_converges_to_done(spy_setters):
direct, done = spy_setters
_make_task("done")
# Window is NOT active (no ARMED sentinel) -> Monitoring is spurious.
for setter in (
plane_sync.set_issue_monitoring,
plane_sync.set_issue_awaiting_deploy,
plane_sync.set_issue_deploying,
):
done.reset_mock()
direct.reset_mock()
setter("ORCH-061")
# Converged to Done; no intermediate deploy-status PATCH.
done.assert_called_once_with("ORCH-061")
direct.assert_not_called()
def test_tc01_decide_verdicts_for_done():
_make_task("done")
# No window -> all three converge.
assert guard.decide("ORCH-061", guard.MONITORING) == guard.CONVERGE_DONE
assert guard.decide("ORCH-061", guard.AWAITING) == guard.CONVERGE_DONE
assert guard.decide("ORCH-061", guard.DEPLOYING) == guard.CONVERGE_DONE
def test_tc01_decide_allows_monitoring_in_active_window(tmp_path, monkeypatch):
_make_task("done")
# Arm the window: ARMED present, DONE absent.
post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed")
assert post_deploy.window_active("orchestrator", "ORCH-061") is True
assert guard.decide("ORCH-061", guard.MONITORING) == guard.ALLOW
# Awaiting/Deploying are ALWAYS spurious for a done task, even with a window.
assert guard.decide("ORCH-061", guard.AWAITING) == guard.CONVERGE_DONE
# Once the window closes (DONE present) Monitoring converges too.
post_deploy.mark_done("orchestrator", "ORCH-061")
assert post_deploy.window_active("orchestrator", "ORCH-061") is False
assert guard.decide("ORCH-061", guard.MONITORING) == guard.CONVERGE_DONE
# --- TC-02 ------------------------------------------------------------------
def test_tc02_idempotent_no_pendulum(spy_setters):
direct, done = spy_setters
_make_task("done")
# Repeated calls keep converging to Done; the intermediate Monitoring PATCH
# never fires, so there is no Done<->deploy-status pendulum.
for _ in range(5):
plane_sync.set_issue_monitoring("ORCH-061")
assert direct.call_count == 0
assert done.call_count == 5 # idempotent PATCH-equivalent (same terminal state)
# --- TC-03 ------------------------------------------------------------------
def test_tc03_non_terminal_not_suppressed(spy_setters):
direct, done = spy_setters
_make_task("deploy") # a really-deploying task
plane_sync.set_issue_awaiting_deploy("ORCH-061")
plane_sync.set_issue_deploying("ORCH-061")
plane_sync.set_issue_monitoring("ORCH-061")
# All three proceed to a real PATCH; nothing converges to Done.
assert direct.call_count == 3
done.assert_not_called()
assert guard.decide("ORCH-061", guard.MONITORING) == guard.ALLOW
# --- TC-04 ------------------------------------------------------------------
def test_tc04_kill_switch(spy_setters, monkeypatch):
direct, done = spy_setters
_make_task("done")
# OFF -> terminal-blind, the monitoring PATCH proceeds (1:1 pre-ORCH-094).
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", False)
plane_sync.set_issue_monitoring("ORCH-061")
assert direct.call_count == 1
done.assert_not_called()
# ON -> converge to Done.
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True)
direct.reset_mock()
done.reset_mock()
plane_sync.set_issue_monitoring("ORCH-061")
direct.assert_not_called()
done.assert_called_once_with("ORCH-061")
# --- TC-05 ------------------------------------------------------------------
def test_tc05_never_raise_on_db_error(spy_setters, monkeypatch):
direct, done = spy_setters
_make_task("done")
def _boom(_wi):
raise RuntimeError("db down")
monkeypatch.setattr(_db, "get_task_by_work_item_id", _boom)
# decide degrades to ALLOW (fail-safe), never raises.
assert guard.decide("ORCH-061", guard.MONITORING) == guard.ALLOW
# The setter proceeds with the normal PATCH (1:1), no convergence, no crash.
plane_sync.set_issue_monitoring("ORCH-061")
assert direct.call_count == 1
done.assert_not_called()
def test_tc05_unknown_task_allows(spy_setters):
direct, done = spy_setters
# No task row at all -> ALLOW (foreign/unknown issue, not ours).
assert guard.decide("ORCH-999", guard.MONITORING) == guard.ALLOW
plane_sync.set_issue_monitoring("ORCH-999")
assert direct.call_count == 1
done.assert_not_called()
def test_tc05_cancelled_is_suppressed(spy_setters):
direct, done = spy_setters
_make_task("cancelled")
assert guard.decide("ORCH-061", guard.MONITORING) == guard.SUPPRESS
plane_sync.set_issue_monitoring("ORCH-061")
# Suppressed: neither an intermediate PATCH nor a Done convergence.
direct.assert_not_called()
done.assert_not_called()
# --- TC-12 ------------------------------------------------------------------
def test_tc12_non_self_repo_inert(spy_setters):
direct, done = spy_setters
# A non-self repo done task: the guard is inert (self-hosting only, empty CSV).
_make_task("done", repo="enduro-trails", wi="ET-042", branch="feature/ET-042-x")
assert guard.applies("enduro-trails") is False
assert guard.decide("ET-042", guard.MONITORING) == guard.ALLOW
plane_sync.set_issue_monitoring("ET-042")
# Behaviour unchanged: the requested PATCH proceeds, no convergence.
assert direct.call_count == 1
done.assert_not_called()
def test_tc12_csv_scope_overrides_self_hosting(monkeypatch):
_make_task("done", repo="enduro-trails", wi="ET-042", branch="feature/ET-042-x")
# Explicit CSV scope brings a non-self repo in-scope.
monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "enduro-trails")
assert guard.applies("enduro-trails") is True
assert guard.applies("orchestrator") is False # not listed -> out of scope
assert guard.decide("ET-042", guard.MONITORING) == guard.CONVERGE_DONE

View File

@@ -135,7 +135,10 @@ def test_tc08_self_deploy_done_sets_monitoring_not_done(monkeypatch):
assert _stage(task_id) == "done"
# Self-hosting: the issue enters the Monitoring window, NOT terminal Done yet.
stage_engine.set_issue_monitoring.assert_called_once_with("ORCH-036")
# ORCH-094: the terminal-sync caller now tags the reason (FR-4 observability).
stage_engine.set_issue_monitoring.assert_called_once_with(
"ORCH-036", reason="advance:deploy->done"
)
stage_engine.set_issue_done.assert_not_called()

View File

@@ -0,0 +1,170 @@
"""ORCH-094 — deterministic post-deploy-monitor termination (FR-3 / AC-3).
Covers (04-test-plan.yaml):
TC-06 after the window finishes (HEALTHY, ticks==budget -> set_issue_done +
`done` marker) there are NO further status PATCHes for the task (a second
tick is a no-op: 0 set_issue_* calls).
TC-07 a tick at DB stage=done with a closed window OR a task cancelled mid-window
-> immediate no-op: no status PATCH and no next-tick enqueue (zombie-tick
excluded).
TC-08 arm_monitor does not re-arm a task already in done (armed/done marker ->
no-op), and a deploy->done re-drive after the window closed converges to
Done instead of resurrecting Monitoring.
"""
import os
import tempfile
import pytest
_test_db = os.path.join(tempfile.gettempdir(), "test_post_deploy_termination.db")
os.environ["ORCH_DB_PATH"] = _test_db
os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
from unittest.mock import MagicMock # noqa: E402
import src.db as _db # noqa: E402
from src.db import init_db, get_db # noqa: E402
from src import stage_engine # noqa: E402
from src import post_deploy # noqa: E402
from src import config as cfg # noqa: E402
@pytest.fixture(autouse=True)
def fresh_db(monkeypatch, tmp_path):
monkeypatch.setattr(_db.settings, "db_path", _test_db)
if os.path.exists(_test_db):
os.unlink(_test_db)
init_db()
monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path))
monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path))
# Small window so the budget is 1 tick (window // interval).
monkeypatch.setattr(stage_engine.settings, "post_deploy_window_s", 10)
monkeypatch.setattr(stage_engine.settings, "post_deploy_interval_s", 10)
monkeypatch.setattr(post_deploy.settings, "post_deploy_window_s", 10)
monkeypatch.setattr(post_deploy.settings, "post_deploy_interval_s", 10)
# write_post_deploy_log touches a worktree/git; stub it.
monkeypatch.setattr(post_deploy, "write_post_deploy_log", MagicMock(return_value=True))
yield
@pytest.fixture
def spy_status(monkeypatch):
setters = {}
for name in ("set_issue_done", "set_issue_monitoring", "set_issue_awaiting_deploy",
"set_issue_deploying", "set_issue_blocked"):
m = MagicMock()
monkeypatch.setattr(stage_engine, name, m)
setters[name] = m
monkeypatch.setattr(stage_engine, "_notify_post_deploy", MagicMock())
return setters
def _make_task(stage="done", repo="orchestrator", wi="ORCH-061", branch="feature/ORCH-061-x"):
conn = get_db()
cur = conn.execute(
"INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
"VALUES (?, ?, ?, ?, ?)",
(f"plane-{wi}", wi, repo, branch, stage),
)
tid = cur.lastrowid
conn.commit()
conn.close()
return tid
def _jobs():
conn = get_db()
rows = conn.execute("SELECT agent FROM jobs ORDER BY id").fetchall()
conn.close()
return [r[0] for r in rows]
def _healthy(*a, **k):
return post_deploy.ProbeResult(health_ok=True, total=2, fivexx=0, detail="ok")
# --- TC-06 ------------------------------------------------------------------
def test_tc06_clean_finish_then_no_more_patches(spy_status, monkeypatch):
monkeypatch.setattr(post_deploy, "probe_signals", _healthy)
tid = _make_task("done")
post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed")
job = {"task_id": tid, "repo": "orchestrator", "id": 1, "agent": "post-deploy-monitor"}
# Tick 1: budget==1, ticks==1 -> HEALTHY window exhausted -> finish.
stage_engine.run_post_deploy_monitor(job)
spy_status["set_issue_done"].assert_called_once_with("ORCH-061")
assert post_deploy.has_marker("orchestrator", "ORCH-061", post_deploy.DONE)
# No next tick was enqueued (window exhausted).
assert _jobs() == []
# Tick 2 (e.g. duplicate job): DONE marker present -> no-op, ZERO new PATCHes.
spy_status["set_issue_done"].reset_mock()
stage_engine.run_post_deploy_monitor(job)
spy_status["set_issue_done"].assert_not_called()
spy_status["set_issue_monitoring"].assert_not_called()
assert _jobs() == []
# --- TC-07 ------------------------------------------------------------------
def test_tc07_cancelled_mid_window_is_noop(spy_status, monkeypatch):
monkeypatch.setattr(post_deploy, "probe_signals", _healthy)
tid = _make_task("cancelled")
post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed")
job = {"task_id": tid, "repo": "orchestrator", "id": 1, "agent": "post-deploy-monitor"}
stage_engine.run_post_deploy_monitor(job)
# Zombie-tick guard: window closed, NO status PATCH, NO next tick.
for name, m in spy_status.items():
m.assert_not_called()
assert post_deploy.has_marker("orchestrator", "ORCH-061", post_deploy.DONE)
assert _jobs() == []
def test_tc07_finished_window_is_noop(spy_status, monkeypatch):
monkeypatch.setattr(post_deploy, "probe_signals", _healthy)
tid = _make_task("done")
# Window already finished (DONE marker present) -> no active basis to tick.
post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed")
post_deploy.mark_done("orchestrator", "ORCH-061")
job = {"task_id": tid, "repo": "orchestrator", "id": 1, "agent": "post-deploy-monitor"}
stage_engine.run_post_deploy_monitor(job)
spy_status["set_issue_done"].assert_not_called()
spy_status["set_issue_monitoring"].assert_not_called()
assert _jobs() == []
# --- TC-08 ------------------------------------------------------------------
def test_tc08_arm_monitor_idempotent_no_rearm(monkeypatch):
tid = _make_task("done")
# First arm: writes ARMED + enqueues tick 1.
assert post_deploy.arm_monitor("orchestrator", "ORCH-061", "feature/ORCH-061-x", tid) is True
assert _jobs() == ["post-deploy-monitor"]
# Second arm (re-drive deploy->done): ARMED present -> no-op, no new job.
assert post_deploy.arm_monitor("orchestrator", "ORCH-061", "feature/ORCH-061-x", tid) is False
assert _jobs() == ["post-deploy-monitor"]
def test_tc08_redrive_after_window_closed_converges(spy_status, monkeypatch):
# Guard ON, self-hosting.
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False)
monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False)
_make_task("done")
# Window armed then closed (a completed post-deploy observation).
post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed")
post_deploy.mark_done("orchestrator", "ORCH-061")
# A stale re-drive calling the REAL guarded setter must converge to Done, not
# resurrect Monitoring. (Use the real plane_sync setter via stage_engine import.)
from src import plane_sync
direct = MagicMock()
done = MagicMock()
monkeypatch.setattr(plane_sync, "_set_issue_state_direct", direct)
monkeypatch.setattr(plane_sync, "set_issue_done", done)
monkeypatch.setattr(plane_sync, "_resolve_project_id", lambda w=None, p=None: "proj-1")
monkeypatch.setattr(plane_sync, "get_project_states", lambda pid: {"monitoring": "S-mon"})
plane_sync.set_issue_monitoring("ORCH-061", reason="advance:deploy->done")
direct.assert_not_called()
done.assert_called_once_with("ORCH-061")

View File

@@ -0,0 +1,82 @@
"""ORCH-094 — sync convergence for a done task stuck on a deploy status (TC-10).
Integration-level: ANY sync source (reconciler tick / monitor tick / a direct
deploy-status setter call) that touches a DB-done task converges Plane to Done
idempotently instead of an intermediate deploy status, and a repeated tick does
NOT swing the Done<->deploy-status pendulum. The guard lives on the setter
(ADR-001 D1/D7), so the reconciler code itself is unchanged — driving the setter
the way a stale actor would is the faithful reproduction of the 061 flapp.
"""
import os
import tempfile
import pytest
_test_db = os.path.join(tempfile.gettempdir(), "test_reconciler_done_converge.db")
os.environ["ORCH_DB_PATH"] = _test_db
os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
from unittest.mock import MagicMock # noqa: E402
import src.db as _db # noqa: E402
from src.db import init_db, get_db # noqa: E402
from src import plane_sync # noqa: E402
from src import post_deploy # noqa: E402
from src import config as cfg # noqa: E402
@pytest.fixture(autouse=True)
def fresh_db(monkeypatch, tmp_path):
monkeypatch.setattr(_db.settings, "db_path", _test_db)
if os.path.exists(_test_db):
os.unlink(_test_db)
init_db()
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False)
monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False)
monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path))
monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path))
yield
@pytest.fixture
def spy(monkeypatch):
direct = MagicMock()
done = MagicMock()
monkeypatch.setattr(plane_sync, "_set_issue_state_direct", direct)
monkeypatch.setattr(plane_sync, "set_issue_done", done)
monkeypatch.setattr(plane_sync, "_resolve_project_id", lambda w=None, p=None: "proj-1")
monkeypatch.setattr(
plane_sync, "get_project_states",
lambda pid: {"awaiting_deploy": "S-aw", "deploying": "S-dep", "monitoring": "S-mon"},
)
return direct, done
def _make_task(stage="done", repo="orchestrator", wi="ORCH-061"):
conn = get_db()
conn.execute(
"INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
"VALUES (?, ?, ?, ?, ?)",
(f"plane-{wi}", wi, repo, "feature/ORCH-061-x", stage),
)
conn.commit()
conn.close()
def test_tc10_repeated_sync_converges_no_pendulum(spy):
direct, done = spy
_make_task("done") # done, window closed (no ARMED sentinel)
# Simulate many sync ticks alternately trying to set Monitoring / Awaiting,
# exactly like the observed 061 pendulum (Awaiting <-> Monitoring forever).
for i in range(10):
if i % 2 == 0:
plane_sync.set_issue_monitoring("ORCH-061", reason="reconciler-tick")
else:
plane_sync.set_issue_awaiting_deploy("ORCH-061", reason="reconciler-tick")
# Every tick converged to Done; not a single intermediate deploy-status PATCH.
assert direct.call_count == 0
assert done.call_count == 10
# All convergence calls target the same terminal Done (no swing).
assert all(c.args == ("ORCH-061",) for c in done.call_args_list)

View File

@@ -0,0 +1,128 @@
"""ORCH-094 — the real deploy cycle is NOT suppressed by the guard (TC-11 / AC-4).
A genuinely-deploying (non-terminal) self-hosting task must still walk
`Awaiting Deploy -> Deploying -> Monitoring after Deploy -> Done` exactly as before
ORCH-094. The critical regression case is the LEGITIMATE first `Monitoring`: by the
time the terminal-sync runs the task is ALREADY DB-`done` (update_task_stage ran
above), so the guard would wrongly converge it to Done UNLESS the arm-block moved
ABOVE the terminal-sync (ADR-001 D3) marks the post-deploy window active first.
This test exercises that ordering end-to-end via run_deploy_finalizer with the REAL
guard + REAL arm_monitor wired in (only the network PATCH primitive is mocked).
"""
import os
import tempfile
import pytest
_test_db = os.path.join(tempfile.gettempdir(), "test_self_deploy_cycle_regression.db")
os.environ["ORCH_DB_PATH"] = _test_db
os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir()
os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token")
os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token")
from unittest.mock import MagicMock # noqa: E402
import src.db as _db # noqa: E402
from src.db import init_db, get_db # noqa: E402
from src import stage_engine # noqa: E402
from src import plane_sync # noqa: E402
from src import post_deploy # noqa: E402
from src import self_deploy # noqa: E402
from src import config as cfg # noqa: E402
@pytest.fixture(autouse=True)
def fresh_db(monkeypatch, tmp_path):
monkeypatch.setattr(_db.settings, "db_path", _test_db)
if os.path.exists(_test_db):
os.unlink(_test_db)
init_db()
monkeypatch.setattr(self_deploy.settings, "repos_dir", str(tmp_path))
monkeypatch.setattr(self_deploy.settings, "host_repos_dir", str(tmp_path))
monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path))
monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path))
# Guard ON, self-hosting only.
monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False)
monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False)
# Post-deploy monitor applies for self repo (arm fires on deploy->done).
monkeypatch.setattr(post_deploy.settings, "post_deploy_monitor_enabled", True)
monkeypatch.setattr(post_deploy.settings, "post_deploy_repos", "")
monkeypatch.setattr(stage_engine.post_deploy.settings, "post_deploy_monitor_enabled", True)
monkeypatch.setattr(stage_engine.post_deploy.settings, "post_deploy_repos", "")
# Stub the worktree/git artefact writers.
monkeypatch.setattr(stage_engine.self_deploy, "write_deploy_log", MagicMock(return_value=True))
monkeypatch.setattr(stage_engine.merge_gate, "release_merge_lease", MagicMock())
yield
@pytest.fixture
def spy_plane(monkeypatch):
"""Spy plane_sync's low-level PATCH + Done convergence (the REAL guard runs)."""
direct = MagicMock()
done = MagicMock()
monkeypatch.setattr(plane_sync, "_set_issue_state_direct", direct)
monkeypatch.setattr(plane_sync, "set_issue_done", done)
monkeypatch.setattr(plane_sync, "_resolve_project_id", lambda w=None, p=None: "proj-1")
monkeypatch.setattr(
plane_sync, "get_project_states",
lambda pid: {"awaiting_deploy": "S-aw", "deploying": "S-dep", "monitoring": "S-mon",
"done": "S-done"},
)
# stage_engine.set_issue_done is a module-level binding -> patch it too so a
# non-self / fallback Done path is observable; here we expect Monitoring though.
monkeypatch.setattr(stage_engine, "set_issue_done", done)
return direct, done
def _make_task(stage, repo="orchestrator", wi="ORCH-063", branch="feature/ORCH-063-x"):
conn = get_db()
cur = conn.execute(
"INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) "
"VALUES (?, ?, ?, ?, ?)",
(f"plane-{wi}", wi, repo, branch, stage),
)
tid = cur.lastrowid
conn.commit()
conn.close()
return tid
def _pass(*a, **k):
return (True, "ok")
def test_tc11_non_terminal_awaiting_deploying_pass(spy_plane):
direct, done = spy_plane
_make_task("deploy")
# Phase A / Phase B statuses on a NON-terminal task proceed (no convergence).
plane_sync.set_issue_awaiting_deploy("ORCH-063", reason="phase_a")
plane_sync.set_issue_deploying("ORCH-063", reason="phase_b")
assert direct.call_count == 2
done.assert_not_called()
def test_tc11_legit_monitoring_preserved_on_deploy_done(spy_plane, monkeypatch):
direct, done = spy_plane
# Hook reported exit 0.
self_deploy.write_marker("orchestrator", "ORCH-063", self_deploy.RESULT, "0")
monkeypatch.setattr(
stage_engine, "QG_CHECKS",
{**stage_engine.QG_CHECKS, "check_deploy_status": _pass},
)
tid = _make_task("deploy")
stage_engine.run_deploy_finalizer(
{"task_id": tid, "repo": "orchestrator", "id": 1, "agent": "deploy-finalizer"}
)
# Stage advanced to done.
conn = get_db()
stage = conn.execute("SELECT stage FROM tasks WHERE id=?", (tid,)).fetchone()[0]
conn.close()
assert stage == "done"
# The arm-block ran BEFORE terminal-sync -> the window is active -> the guard
# ALLOWS the legitimate Monitoring PATCH (S-mon), it is NOT converged to Done.
assert post_deploy.has_marker("orchestrator", "ORCH-063", post_deploy.ARMED)
mon_calls = [c for c in direct.call_args_list if c.args[1] == "S-mon"]
assert len(mon_calls) == 1, f"expected one Monitoring PATCH, got {direct.call_args_list}"
done.assert_not_called()