From 3738888601d3dc83d09038c244f64db27d5c8bb6 Mon Sep 17 00:00:00 2001 From: claude-bot Date: Tue, 9 Jun 2026 23:31:30 +0300 Subject: [PATCH] fix(deploy): terminal-window-aware guard so done tasks hold Done in Plane (ORCH-094) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A DB stage=done task with 0 active jobs flapped in Plane between `Awaiting Deploy` and `Monitoring after Deploy` instead of holding `Done` (verified live on ORCH-061, task 47): the three deploy-phase setters were terminal-blind, so any stale/duplicate/unknown caller under the bot token re-stamped an intermediate status over the terminal Done, forever. - New leaf src/deploy_status_guard.py (pure, never-raise, config-gated): decide() -> ALLOW | CONVERGE_DONE | SUPPRESS on the entry of set_issue_awaiting_deploy / set_issue_deploying / set_issue_monitoring. A deploy-phase status is legitimate iff the task is non-terminal OR (done AND post-deploy window active); otherwise done converges to Done idempotently, cancelled is suppressed (FR-2, D1/D2). - D3: move post_deploy.arm_monitor ABOVE the terminal-sync block in advance_stage so window_active is True when the legitimate first Monitoring is set (the task is already DB-done by then); a re-drive after the window closes converges to Done. - D4: run_post_deploy_monitor no-ops without a status PATCH / re-queue when the task became cancelled mid-window (zombie-tick guard, FR-3). - D5: additive `reason` kwarg on the three setters + one structured log line per verdict (work_item/caller/target/db_stage/window_active/verdict); new read-only db.get_task_by_work_item_id; post_deploy.window_active helper. - Flags deploy_status_guard_enabled (kill-switch -> 1:1) / deploy_status_guard_repos (CSV; empty = self-hosting only). STAGE_TRANSITIONS / QG_CHECKS / check_* / machine-verdict keys / DB schema untouched (reads existing tasks.stage). Tests: TC-01..TC-12 across 5 new test modules + config flags; updated the reason-kwarg assertions in test_deploy_terminal_sync / test_deploy_approve. Full regress green (1413). Docs: CHANGELOG, CLAUDE.md, docs/architecture/README.md (status -> реализовано), .env.example. Refs: ORCH-094 Co-Authored-By: Claude Opus 4.8 --- .env.example | 11 + CHANGELOG.md | 8 + CLAUDE.md | 2 + docs/architecture/README.md | 2 +- src/config.py | 26 +++ src/db.py | 22 ++ src/deploy_status_guard.py | 191 +++++++++++++++ src/plane_sync.py | 41 +++- src/post_deploy.py | 22 ++ src/stage_engine.py | 65 ++++-- tests/test_config.py | 27 +++ tests/test_deploy_approve.py | 6 +- tests/test_deploy_status_observability.py | 88 +++++++ tests/test_deploy_status_terminal_guard.py | 217 ++++++++++++++++++ tests/test_deploy_terminal_sync.py | 5 +- tests/test_post_deploy_monitor_termination.py | 170 ++++++++++++++ ...test_reconciler_done_deploy_convergence.py | 82 +++++++ tests/test_self_deploy_cycle_regression.py | 128 +++++++++++ 18 files changed, 1088 insertions(+), 25 deletions(-) create mode 100644 src/deploy_status_guard.py create mode 100644 tests/test_deploy_status_observability.py create mode 100644 tests/test_deploy_status_terminal_guard.py create mode 100644 tests/test_post_deploy_monitor_termination.py create mode 100644 tests/test_reconciler_done_deploy_convergence.py create mode 100644 tests/test_self_deploy_cycle_regression.py diff --git a/.env.example b/.env.example index dfba7c6..90c59c4 100644 --- a/.env.example +++ b/.env.example @@ -139,6 +139,17 @@ ORCH_SERIAL_GATE_FREEZE_ENABLED=true # for enduro too). ORCH_STOP_STATUS_ENABLED=true ORCH_STOP_STATUS_REPOS= +# ORCH-094: terminal-window-aware guard for the three deploy-phase Plane status +# setters (set_issue_awaiting_deploy / set_issue_deploying / set_issue_monitoring). +# A DB stage=done task converges to Done idempotently instead of flapping +# Awaiting <-> Monitoring, EXCEPT the legitimate post-deploy Monitoring while the +# window is active (ARMED & not DONE). Leaf src/deploy_status_guard.py, never-raise; +# STAGE_TRANSITIONS / QG_CHECKS / machine-verdict keys untouched (no DB migration). +# DEPLOY_STATUS_GUARD_ENABLED=false -> setters are terminal-blind (1:1 pre-ORCH-094). +# DEPLOY_STATUS_GUARD_REPOS (CSV) -> scope; EMPTY = self-hosting only (orchestrator), +# the only repo where deploy-phase statuses are set. +ORCH_DEPLOY_STATUS_GUARD_ENABLED=true +ORCH_DEPLOY_STATUS_GUARD_REPOS= # ORCH-071/073: merge-verify under-gate on the `deploy -> done` edge (врезка in # advance_stage, NOT a new STAGE_TRANSITIONS edge / registered QG). A deterministic # merge-actor merges the feature code-PR via the Gitea PR-merge API (never push/ diff --git a/CHANGELOG.md b/CHANGELOG.md index e0721df..624f669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ Формат: [Keep a Changelog](https://keepachangelog.com/). Записи — на смысловой PR/задачу. ## [Unreleased] +- **Терминальная (done) задача держит `Done` в Plane: terminal-window-aware гард deploy-статусов** (ORCH-094, `fix`): задача с БД `stage=done` и 0 активных job'ов (верифицировано на ORCH-061, task 47) стабильно флаппила в Plane `Awaiting Deploy ⟷ Monitoring after Deploy` (273 активности парами, само не затихает) вместо `Done`. Корень: три deploy-фазовых сеттера (`set_issue_awaiting_deploy`/`set_issue_deploying`/`set_issue_monitoring`) **терминал-слепы** — любой стейл/двойной/неизвестный вызов под бот-токеном перезаписывает `Done` промежуточным deploy-статусом, и обратно, бесконечно. **Аддитивно, never-raise, под kill-switch, в зоне self-hosting:** `STAGE_TRANSITIONS` / `QG_CHECKS` / `check_*` / machine-verdict ключи (`deploy_status:`/`staging_status:`/…) / схема БД — **не тронуты** (читается существующая `tasks.stage`, без миграции). + - **Единый гард на низком чокпоинте (FR-2, D1/D2):** новый leaf `src/deploy_status_guard.py` (чистая, never-raise, config-gated логика; по образцу `serial_gate.py`/`labels.py`/`cancel.py`) — `decide(work_item_id, target, reason) -> ALLOW | CONVERGE_DONE | SUPPRESS`. Гард ставится на **входе** трёх сеттеров `plane_sync` (а не в caller'ах `stage_engine`) → перехватывает **любой** путь, включая неизвестный актор под бот-токеном. Предикат легитимности: deploy-статус легитимен ⇔ задача **нетерминальна** ИЛИ (`done` **И** активно пост-деплой-окно `post_deploy.window_active` = ARMED & не DONE). Для `done`: `monitoring`+окно-активно → `ALLOW`; иначе → `CONVERGE_DONE` (сеттер вместо PATCH'а зовёт `set_issue_done`, идемпотентно). `cancelled` → `SUPPRESS` (не штампуем поверх терминала ORCH-090). Нетерминальная задача → `ALLOW` (рабочий deploy-цикл 1:1, AC-4). Task не найден / не-self репо / kill-switch off / любое исключение → `ALLOW` (fail-safe к прежнему поведению 1:1, NFR-1). + - **Перенос арм-блока перед terminal-sync (D3, AC-4):** в `advance_stage` (ветка `next_stage=="done"`) блок `post_deploy.arm_monitor` перемещён **выше** блока `set_issue_monitoring` (стр. 404). Критично: `update_task_stage(task_id,"done")` пишет `stage='done'` **раньше** легитимного первого `Monitoring` — без переноса гард ошибочно свёл бы его к Done. Арм-первым пишет `ARMED` → `window_active==True` → `ALLOW` пропускает легитимный `Monitoring`; re-drive `deploy→done` **после** закрытия окна (`DONE` present) → `window_active==False` → `CONVERGE_DONE` (не воскрешает `Monitoring`). Перенос безопасен: `arm_monitor` лишь пишет sentinel + ставит отложенный job, не зависит от Plane-статуса/merge-lease (release остаётся после terminal-sync). Инварианты ORCH-021 (идемпотентный арм по `ARMED`) и ORCH-066 (`deploy→done` self ⇒ `Monitoring`) сохранены. + - **Харднинг пост-деплой-монитора (FR-3, D4, AC-3):** `run_post_deploy_monitor` — существующий идемпотентный страж `has_marker(DONE)` (no-op завершённого окна) сохранён; аддитивно: тик при БД `stage='cancelled'` мид-окно → закрыть окно `mark_done` **без статус-PATCH и без перепостановки** следующего тика (zombie-tick guard). Перепостановка остаётся строго при `HEALTHY and ticks < budget` (тик ≡ job; нет job → нет тика). После закрытия окна — 0 последующих статус-PATCH; любой стейл `set_issue_monitoring` добивается гардом D2. + - **Наблюдаемость (FR-4, D5, AC-5):** аддитивный BC-kwarg `reason: str | None = None` у трёх сеттеров; call-site'ы передают `"advance:deploy->done"`/`"phase_a"`/`"phase_b"`. `decide` эмитит ОДНУ структурную запись на вызов: `work_item`, `caller(reason)`, `target_status`, `db_stage`, `window_active`, `verdict` (`ALLOW` → INFO; `CONVERGE_DONE`/`SUPPRESS` → WARNING, «что подавили и почему» — атрибуция будущего флаппа). Новый read-only аксессор `db.get_task_by_work_item_id` (human-readable `work_item_id` матчит живой ряд; тумбстоны ORCH-090 имеют суффикс `#cancelled-`). + - **Конфиг/откат (FR-5, D6):** `src/config.py` `deploy_status_guard_enabled: bool = True` (env `ORCH_DEPLOY_STATUS_GUARD_ENABLED`; `False` → сеттеры терминал-слепы, поведение **1:1** прежнее) / `deploy_status_guard_repos: str = ""` (env `ORCH_DEPLOY_STATUS_GUARD_REPOS`; CSV, **пусто → self-hosting only** — не-self репо (enduro) гард не трогает, нулевая регрессия). Откат: `ORCH_DEPLOY_STATUS_GUARD_ENABLED=false` (мгновенный runtime) или revert ветки. + - **Источник флаппа (BR-7):** code-писатели deploy-статусов — только `stage_engine.py:404/1218/1316`; реконсилятор F-2 эти статусы не перебирает; live-overlay `notifications.py` — read-only. Гард — **буфер на стороне орка**, гасящий маятник за один цикл независимо от актора (известный/стейл/неизвестный под бот-токеном). Если актор — внешняя Plane-automation под другим токеном, code-фикс не закрывает её полностью, но идемпотентное схождение к Done нейтрализует видимый эффект. + - **Трассировка:** перед правкой блока `next_stage=="done"` (маркеры ORCH-021/066/043/088) прочитаны их ADR — инварианты сохранены (deploy→done self ⇒ Monitoring; монитор-close ⇒ Done; терминал-набор `{done,cancelled}`). Тесты: `tests/test_deploy_status_terminal_guard.py` (TC-01..05/12), `tests/test_post_deploy_monitor_termination.py` (TC-06..08), `tests/test_deploy_status_observability.py` (TC-09), `tests/test_reconciler_done_deploy_convergence.py` (TC-10), `tests/test_self_deploy_cycle_regression.py` (TC-11). Обновлены анти-регресс-ассерты `tests/test_deploy_terminal_sync.py`/`test_deploy_approve.py` под `reason`-kwarg. Полный регресс `tests/ -q` зелёный (1411). ADR: `docs/work-items/ORCH-094/06-adr/ADR-001-terminal-window-aware-deploy-status-guard.md`, сквозной `docs/architecture/adr/adr-0028-terminal-window-aware-deploy-status-guard.md`. - **Merge-актор ретраит транзиентные ошибки Gitea (405/5xx) + гард «ветка уже в `main`»** (ORCH-093, `fix`): две точечные доработки детерминированного merge-актора `src/merge_gate.py`, чинящие инцидент **ORCH-063**: self-deploy прошёл, staging OK, PR был `open`+`mergeable`, но `POST /pulls/{n}/merge` вернул `HTTP 405 "Please try again later"` (Gitea пересчитывал `mergeable` сразу после пуша) → one-shot `merge_pr` мгновенно вернул `False` → корректная защита ORCH-071/081 удержала задачу на `deploy` + потребовала ручной домерж; повторный прогон финализатора плодил мусорный пустой PR. **Аддитивно, never-raise, под существующими kill-switch'ами:** `STAGE_TRANSITIONS` / `QG_CHECKS` / схема БД — **не тронуты**; INV-4 (мерж только через Gitea PR-merge API, никогда `push`/`force-push` в `main`) сохранён 1:1. - **Retry-loop транзиента (FR-1/FR-2, AC-1/AC-2/AC-3, D1/D2):** `merge_pr` оборачивает **только** мутирующий `POST …/merge` в ограниченный retry-loop с экспоненциальным backoff (`min(base*2^(i-1), max)`, дефолты 2/5 с → суммарный сон `(N-1)*max ≤ 10 с`, monitor-поток не подвешивается). Классификатор `_classify_merge_response`: **транзиент** (ретрай) — `405`/`408`/любой `5xx`/`httpx`-таймаут/сетевая ошибка, **и** `409`/`422` когда PR всё ещё mergeable; **терминал** (быстрый честный `False`, защита ORCH-071/081 как прежде) — `403`/`404`/реальный конфликт (`409`/`422` при `mergeable==False`). Неоднозначный `409`/`422` разрешается доп. `GET /pulls/{index}` → `mergeable`; дефолт-политика `mergeable==None`/недоступно → **транзиент** (fail-OPEN-в-ретрай: икота Gitea — наблюдаемый кейс, бюджет конечен, backstop сохранён). Каждая попытка логируется `attempt i/N` (образец `check_ci_green`). - **Гард already-in-main (FR-3/FR-4, AC-4, D3/D4):** новый leaf `_branch_fully_in_main` (`git merge-base --is-ancestor HEAD origin/main` в per-branch worktree) вызывается в `ensure_open_pr` **между** «открытый code-PR не найден» и `POST …/pulls`: ветка целиком в `main` (нет коммитов `origin/main..HEAD`) → новый исход `"already-in-main"` **без создания PR** (нет мусорного пустого PR на уже влитой ветке). git-ошибка/ambiguous (`None`) → **fail-OPEN** (деградация на create-путь, НЕ ложный no-op). В `stage_engine._handle_merge_verify` исход `already-in-main` **пропускает** `merge_pr` (мержить нечего) и отдаёт авторитетному SHA-in-main (`verify_merged_to_main`) довести до `done`; это НЕ HOLD. SHA-in-main остаётся единственным доказательством мержа (ADR-0014). diff --git a/CLAUDE.md b/CLAUDE.md index dfb0be4..0035cb0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,6 +41,8 @@ created → analysis → architecture → development → review → testing → ## Статусная модель Plane (ORCH-066) — индикация ≠ управление Статусы Plane — это **слой B (индикация)**, отдельный от **слоя A (машина стадий)** `src/stages.py::STAGE_TRANSITIONS`. Plane показывает наблюдателю осмысленную картину (`Backlog → Todo → Analysis → Architecture → Development → Code-Review → Testing → Awaiting Deploy → Deploying → Monitoring after Deploy → Done` + человеческие гейты `In Review/Approved`, `Confirm Deploy`), но НИКОГДА не управляет конвейером. Маппинг и сеттеры — `src/plane_sync.py` (6 новых ключей: `to_analyse/analysis/code_review/awaiting_deploy/deploying/monitoring`), с project-relative alias-fallback: на частично сконфигурированном проекте новый ключ деградирует на базовый UUID ТОГО ЖЕ проекта (нулевая регрессия для enduro-trails). Детали — `docs/architecture/README.md`. +**Terminal-window-aware гард deploy-статусов (ORCH-094).** Задача с БД `stage=done` и 0 активных job'ов стабильно держит Plane=`Done`: три deploy-фазовых сеттера (`set_issue_awaiting_deploy`/`set_issue_deploying`/`set_issue_monitoring`) были терминал-слепы и флаппили `Awaiting ⟷ Monitoring` (верифицировано на ORCH-061, task 47), т.к. любой стейл/двойной/неизвестный вызов под бот-токеном перезаписывал терминал промежуточным статусом. Новый leaf `src/deploy_status_guard.py` (чистая, never-raise, config-gated; по образцу `serial_gate`/`labels`/`cancel`) — `decide(work_item_id, target, reason) -> ALLOW | CONVERGE_DONE | SUPPRESS` на **входе** трёх сеттеров `plane_sync` (низкий чокпоинт ловит любой путь, включая неизвестный актор). Инвариант: deploy-статус легитимен ⇔ задача **нетерминальна** ИЛИ (`done` И активно пост-деплой-окно `post_deploy.window_active` = ARMED & не DONE); иначе для `done` — идемпотентное `CONVERGE_DONE` (сеттер зовёт `set_issue_done`), для `cancelled` — `SUPPRESS`. Чтобы легитимный первый `Monitoring` (БД уже `done` к моменту стр. 404) прошёл, арм-блок `post_deploy.arm_monitor` **перенесён выше** terminal-sync-блока в `advance_stage` (ADR-001 D3) → `window_active==True` до выставления `Monitoring`. Монитор-тик при БД `cancelled` мид-окно → закрыть окно без статус-PATCH (zombie-tick guard, FR-3). Наблюдаемость: BC-kwarg `reason` у трёх сеттеров + одна структурная лог-запись на вердикт (`work_item`/`caller`/`target`/`db_stage`/`window_active`/`verdict`; converge/suppress → WARNING). Read-only аксессор `db.get_task_by_work_item_id`. Флаги `deploy_status_guard_enabled` (kill-switch; `False` → 1:1 прежнее) / `deploy_status_guard_repos` (CSV; **пусто → self-hosting only**, enduro не затронут). `STAGE_TRANSITIONS`/`QG_CHECKS`/`check_*`/machine-verdict/схема БД — не тронуты. Детали — `docs/work-items/ORCH-094/06-adr/ADR-001-terminal-window-aware-deploy-status-guard.md`, сквозной `docs/architecture/adr/adr-0028-terminal-window-aware-deploy-status-guard.md`. + ## Нотификации / Telegram live-tracker (ORCH-042/066/067/087) Каждая задача = **одна карточка** в Telegram (`src/notifications.py`). Поведение карточки: - **Дефолт `tracker_mode` — `bump`** (ORCH-067; `edit` доступен через `ORCH_TRACKER_MODE=edit`). diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 76b9dac..08744a5 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -585,7 +585,7 @@ sentinel-state, `write_post_deploy_log`. Подробнее: [adr-0010](adr/adr-0010-post-deploy-monitor.md), детально — `docs/work-items/ORCH-021/06-adr/ADR-001-post-deploy-monitor.md`. -### Terminal-window-aware гард deploy-статусов: done-задача держит Done (ORCH-094 — design) +### Terminal-window-aware гард deploy-статусов: done-задача держит Done (ORCH-094 — реализовано) Терминальная (`done`) задача в Plane **не держала `Done`**: непрерывный флапп `Awaiting Deploy ⟷ Monitoring after Deploy` (верифицировано на **ORCH-061**, task 47, done с 07.06 — 273 активности, само не затихает). Причина: три code-писателя deploy-фазовых статусов diff --git a/src/config.py b/src/config.py index f1313ea..dc48aa9 100644 --- a/src/config.py +++ b/src/config.py @@ -649,6 +649,32 @@ class Settings(BaseSettings): stop_status_enabled: bool = True stop_status_repos: str = "" + # ORCH-094: terminal-window-aware guard for deploy-phase Plane status setters. + # A task with DB stage='done' (and 0 active jobs) was flapping in Plane between + # `Awaiting Deploy` and `Monitoring after Deploy` instead of holding `Done`, + # because the three deploy-phase setters (set_issue_awaiting_deploy / + # set_issue_deploying / set_issue_monitoring) are terminal-blind: any stale / + # duplicate / unknown caller under the bot token re-stamps an intermediate + # deploy status over the terminal Done. ORCH-094 puts a single low choke-point + # guard on the entry of those three setters (leaf src/deploy_status_guard.py): + # for a task whose DB stage is terminal it converges to Done idempotently + # (CONVERGE_DONE), EXCEPT the legitimate post-deploy `Monitoring` while the + # window is still active (ARMED & not DONE). Additive, never-raise; reads the + # existing tasks.stage (no migration); STAGE_TRANSITIONS / QG_CHECKS / + # machine-verdict keys are NOT touched. See + # docs/work-items/ORCH-094/06-adr/ADR-001-terminal-window-aware-deploy-status-guard.md + # and the cross-cutting docs/architecture/adr/adr-0028-…md. + # deploy_status_guard_enabled -> kill-switch (env ORCH_DEPLOY_STATUS_GUARD_ENABLED). + # False -> the setters are terminal-blind, behaviour + # strictly 1:1 as before ORCH-094 (zero regression). + # deploy_status_guard_repos -> CSV scope (env ORCH_DEPLOY_STATUS_GUARD_REPOS). + # Empty -> applies ONLY to the self-hosting repo + # (orchestrator), where deploy-phase statuses are set + # at all; non-empty -> only the listed repos. Tokens + # are sanitised (^[A-Za-z0-9._-]+$) by the guard leaf. + deploy_status_guard_enabled: bool = True + deploy_status_guard_repos: str = "" + # ORCH-073 (ADR-001 Р-4): main-integrity regression guard. After the merge-verify # under-gate confirms the deployed SHA is an ancestor of origin/main (FR-1), a # secondary deterministic (no-LLM) guard checks that a declarative set of markers diff --git a/src/db.py b/src/db.py index a458e62..6701034 100644 --- a/src/db.py +++ b/src/db.py @@ -223,6 +223,28 @@ def get_task_by_plane_id(plane_id: str) -> dict | None: return None +def get_task_by_work_item_id(work_item_id: str) -> dict | None: + """ORCH-094: read-only lookup of the live task row by human-readable + ``work_item_id`` (e.g. ``"ORCH-061"``). + + ``get_task_by_plane_id`` matches the Plane UUIDs (``plane_id`` / + ``plane_issue_id``), not the human-readable ``work_item_id`` the deploy-phase + setters receive — hence this thin accessor. A live row matches exactly; the + ORCH-090 cancel tombstones carry a ``#cancelled-`` suffix on + ``work_item_id`` so they never collide with a clean id. No schema change. + """ + if not work_item_id: + return None + conn = get_db() + try: + row = conn.execute( + "SELECT * FROM tasks WHERE work_item_id = ?", (work_item_id,) + ).fetchone() + finally: + conn.close() + return dict(row) if row else None + + def get_task_by_repo_branch(repo: str, branch: str) -> dict | None: """Find task by repo and branch name.""" conn = get_db() diff --git a/src/deploy_status_guard.py b/src/deploy_status_guard.py new file mode 100644 index 0000000..d0a46e1 --- /dev/null +++ b/src/deploy_status_guard.py @@ -0,0 +1,191 @@ +"""ORCH-094: terminal-window-aware guard for deploy-phase Plane status setters. + +Leaf module — pure, never-raise, config-gated logic over the existing ``tasks`` +table and the restart-safe post-deploy sentinels. Mirrors the leaf pattern of +``src/serial_gate.py`` / ``src/labels.py`` / ``src/cancel.py``: it imports only +``config`` (and lazily ``db`` / ``post_deploy`` / ``qg.checks``), never +``plane_sync`` / ``stage_engine`` — the setters that need a verdict call +:func:`decide`, they do not live here. + +The bug (verified live on ORCH-061, task 47, done since 07.06): a task with DB +``stage='done'`` and no active job flaps in Plane between ``Awaiting Deploy`` and +``Monitoring after Deploy`` instead of holding ``Done``. The three deploy-phase +setters (``set_issue_awaiting_deploy`` / ``set_issue_deploying`` / +``set_issue_monitoring``) are **terminal-blind**: any stale / duplicate / unknown +caller under the bot token re-stamps an intermediate deploy status over the +terminal Done, and the pendulum never settles. + +The fix is a single low choke-point on the entry of those three setters. For a +task whose DB stage is terminal the verdict converges to ``Done`` idempotently, +EXCEPT the one legitimate case: the post-deploy ``Monitoring`` status while the +observation window is still active (``post_deploy.window_active`` — ARMED & not +DONE). The deploy ``Awaiting``/``Deploying`` statuses are ALWAYS spurious for a +``done`` task (Phase A/B happen strictly BEFORE ``deploy -> done``). + +Key invariant (ADR-001 D2): a deploy-phase status is legitimate iff the task is +non-terminal OR (``done`` AND the post-deploy window is active); otherwise the +verdict is idempotent convergence to ``Done`` (for ``done``) / suppression (for +``cancelled``). + +never-raise contract (self-hosting safety): any error / inability to determine +the DB stage degrades to ``ALLOW`` (fail-safe to the prior 1:1 behaviour, NFR-1) +— a local SQLite read is reliable, so in the normal case the stage is read and +the pendulum cannot arise. +""" +from __future__ import annotations + +import logging +import re + +from .config import settings + +logger = logging.getLogger("orchestrator.deploy_status_guard") + +# Verdicts returned by decide() (the setter executes them). +ALLOW = "ALLOW" # PATCH the requested deploy-phase status (normal path). +CONVERGE_DONE = "CONVERGE_DONE" # set_issue_done instead (idempotent convergence). +SUPPRESS = "SUPPRESS" # do nothing (do not stamp over a `cancelled` terminal). + +# Deploy-phase target tokens (one per guarded setter). +AWAITING = "awaiting" +DEPLOYING = "deploying" +MONITORING = "monitoring" + +# Terminal DB stages (harmonised with serial_gate / adr-0026). +_TERMINAL = ("done", "cancelled") + +# Repo tokens embedded into config CSV must match this (mirrors serial_gate R-6). +_REPO_TOKEN = re.compile(r"^[A-Za-z0-9._-]+$") + + +# --------------------------------------------------------------------------- +# Conditionality (mirrors post_deploy_applies / _merge_gate_applies) +# --------------------------------------------------------------------------- +def _scope_repos() -> set[str]: + """Sanitised set of in-scope repo tokens from ``deploy_status_guard_repos``. + + Empty/blank CSV -> empty set, meaning "self-hosting only" (resolved by the + caller via :func:`applies`). Invalid tokens (regex miss) are dropped. Never + raises. + """ + try: + raw = (settings.deploy_status_guard_repos or "").strip() + except Exception: # noqa: BLE001 + return set() + if not raw: + return set() + out: set[str] = set() + for tok in raw.split(","): + t = tok.strip() + if t and _REPO_TOKEN.match(t): + out.add(t) + elif t: + logger.warning("deploy_status_guard: dropping invalid repo token %r", t) + return out + + +def applies(repo: str) -> bool: + """Whether the guard is REAL for this repo (D6). + + * ``deploy_status_guard_enabled=False`` -> always False (kill-switch; the + setters are terminal-blind, 1:1 as before ORCH-094). + * ``deploy_status_guard_repos`` (CSV) non-empty -> real only for listed repos. + * empty CSV -> real ONLY for the self-hosting repo (``orchestrator``), where + deploy-phase statuses are set at all. Mirrors the ORCH-35/36/43/58 + self-hosting-only rollout -> non-self repos (enduro-trails) are untouched + (they never see Awaiting/Deploying/Monitoring; terminal-sync goes straight + to Done), i.e. zero regression. + Never raises -> False on error (degrade to "guard inert"). + """ + try: + if not getattr(settings, "deploy_status_guard_enabled", False): + return False + scope = _scope_repos() + if scope: + return (repo or "").strip() in scope + # Lazy import keeps this module a leaf (avoid importing qg at load time). + from .qg.checks import is_self_hosting_repo + return is_self_hosting_repo(repo) + except Exception as e: # noqa: BLE001 - never-raise + logger.warning("deploy_status_guard.applies error for %s: %s", repo, e) + return False + + +# --------------------------------------------------------------------------- +# Verdict (the single predicate — ADR-001 D2) +# --------------------------------------------------------------------------- +def decide(work_item_id: str, target_status: str, reason: str | None = None) -> str: + """Decide what a deploy-phase setter should do for ``work_item_id`` (D2). + + Returns one of :data:`ALLOW` / :data:`CONVERGE_DONE` / :data:`SUPPRESS`. + Steps (ADR-001 D2): + + 1. kill-switch off -> ALLOW (behaviour 1:1). + 2. task not found -> ALLOW (foreign/unknown issue). + 3. guard not applicable for the repo -> ALLOW (non-self / out-of-scope). + 4. DB stage non-terminal -> ALLOW (live deploy cycle, AC-4). + 5. DB stage == 'cancelled' -> SUPPRESS (do not stamp over it). + 6. DB stage == 'done': + * target == 'monitoring' AND window active -> ALLOW (legit post-deploy). + * otherwise -> CONVERGE_DONE. + 7. any exception / undeterminable stage -> ALLOW (fail-safe, NFR-1). + + Always emits exactly one structured observability line (FR-4 / D5): work_item, + caller (``reason``), target_status, db_stage, window_active, verdict. + """ + db_stage = None + window = None + verdict = ALLOW + try: + if not getattr(settings, "deploy_status_guard_enabled", False): + return ALLOW # step 1 (logged in finally) + + from . import db + task = db.get_task_by_work_item_id(work_item_id) + if task is None: + return ALLOW # step 2 + + repo = task.get("repo") + if not applies(repo): + return ALLOW # step 3 + + db_stage = (task.get("stage") or "").strip() + if db_stage not in _TERMINAL: + verdict = ALLOW # step 4 — non-terminal: legit working deploy cycle + return verdict + + if db_stage == "cancelled": + verdict = SUPPRESS # step 5 + return verdict + + # step 6 — db_stage == 'done' + if target_status == MONITORING: + from . import post_deploy + window = post_deploy.window_active(repo, work_item_id) + if window: + verdict = ALLOW + return verdict + verdict = CONVERGE_DONE + return verdict + except Exception as e: # noqa: BLE001 - never-raise; fail-safe to ALLOW + logger.warning( + "deploy_status_guard.decide error for %s (target=%s) -> ALLOW: %s", + work_item_id, target_status, e, + ) + verdict = ALLOW + return verdict + finally: + # FR-4 / D5: one structured line per call. Convergence/suppression is the + # interesting case — log it at WARNING so a future flapp is easy to attribute. + try: + msg = ( + "deploy_status_guard: work_item=%s caller=%s target=%s db_stage=%s " + "window_active=%s verdict=%s" + ) + argv = (work_item_id, reason, target_status, db_stage, window, verdict) + if verdict == ALLOW: + logger.info(msg, *argv) + else: + logger.warning(msg, *argv) + except Exception: # noqa: BLE001 - logging must never raise + pass diff --git a/src/plane_sync.py b/src/plane_sync.py index 88f651e..9507f29 100644 --- a/src/plane_sync.py +++ b/src/plane_sync.py @@ -951,32 +951,67 @@ def set_issue_code_review(work_item_id: str, project_id: str = None): _set_issue_state_direct(work_item_id, state_id, project_id) -def set_issue_awaiting_deploy(work_item_id: str, project_id: str = None): +def _deploy_status_guarded(work_item_id: str, target: str, reason: str | None) -> bool: + """ORCH-094: apply the terminal-window-aware guard for a deploy-phase setter. + + Returns True iff the caller should PROCEED with the normal PATCH (verdict + ALLOW). On CONVERGE_DONE it drives the task to terminal ``Done`` here (the + idempotent convergence target) and returns False; on SUPPRESS it does nothing + and returns False. never-raise: any error degrades to ALLOW (proceed), keeping + behaviour 1:1 with pre-ORCH-094 (the guard leaf itself fails safe to ALLOW). + """ + try: + from . import deploy_status_guard + verdict = deploy_status_guard.decide(work_item_id, target, reason=reason) + if verdict == deploy_status_guard.CONVERGE_DONE: + set_issue_done(work_item_id) + return False + if verdict == deploy_status_guard.SUPPRESS: + return False + return True + except Exception as e: # noqa: BLE001 - never-raise; proceed (1:1) on doubt + logger.warning(f"deploy_status_guard wrapper error for {work_item_id}: {e}") + return True + + +def set_issue_awaiting_deploy(work_item_id: str, project_id: str = None, reason: str = None): """ORCH-066: set issue to 'Awaiting Deploy' — self-deploy Phase A approval-pending. Degrades to the project's In Review UUID when 'Awaiting Deploy' is not created. + ORCH-094: terminal-window-aware — a task whose DB stage is terminal converges to + Done instead of stamping a spurious deploy status (``reason`` = caller, FR-4). """ + if not _deploy_status_guarded(work_item_id, "awaiting", reason): + return project_id = _resolve_project_id(work_item_id, project_id) state_id = get_project_states(project_id)["awaiting_deploy"] _set_issue_state_direct(work_item_id, state_id, project_id) -def set_issue_deploying(work_item_id: str, project_id: str = None): +def set_issue_deploying(work_item_id: str, project_id: str = None, reason: str = None): """ORCH-066: set issue to 'Deploying' — self-deploy Phase B prod deploy in flight. Degrades to the project's In Progress UUID when 'Deploying' is not created. + ORCH-094: terminal-window-aware (see :func:`set_issue_awaiting_deploy`). """ + if not _deploy_status_guarded(work_item_id, "deploying", reason): + return project_id = _resolve_project_id(work_item_id, project_id) state_id = get_project_states(project_id)["deploying"] _set_issue_state_direct(work_item_id, state_id, project_id) -def set_issue_monitoring(work_item_id: str, project_id: str = None): +def set_issue_monitoring(work_item_id: str, project_id: str = None, reason: str = None): """ORCH-066: set issue to 'Monitoring after Deploy' — post-deploy window open. Degrades to the project's Done UUID when 'Monitoring after Deploy' is not created (so the board shows Done, exactly as before ORCH-066). + ORCH-094: terminal-window-aware — the LEGITIMATE first Monitoring (DB already + ``done`` by the time line 404 runs, but the post-deploy window is active) is + allowed; a stale Monitoring after the window has closed converges to Done. """ + if not _deploy_status_guarded(work_item_id, "monitoring", reason): + return project_id = _resolve_project_id(work_item_id, project_id) state_id = get_project_states(project_id)["monitoring"] _set_issue_state_direct(work_item_id, state_id, project_id) diff --git a/src/post_deploy.py b/src/post_deploy.py index 75afe42..cd86b49 100644 --- a/src/post_deploy.py +++ b/src/post_deploy.py @@ -316,6 +316,28 @@ def has_marker(repo: str, work_item_id: str | None, name: str) -> bool: return False +def window_active(repo: str, work_item_id: str | None) -> bool: + """ORCH-094: True iff a post-deploy observation window is currently OPEN. + + A window is open iff it has been armed (``ARMED`` sentinel) and has NOT yet + finished (no ``DONE`` sentinel). The terminal-window-aware deploy-status guard + (``deploy_status_guard.decide``) uses this to keep the legitimate post-deploy + ``Monitoring after Deploy`` status for a task that is already DB-``done`` while + its window is live, and to converge to ``Done`` once the window has closed. + + Restart-safe (the sentinels live on disk) and never-raise -> False on error + (a doubt resolves to "window closed", i.e. converge to Done — the safe-for- + indication default that matches the bug we are fixing). + """ + try: + return has_marker(repo, work_item_id, ARMED) and not has_marker( + repo, work_item_id, DONE + ) + except Exception as e: # noqa: BLE001 - never-raise + logger.warning("window_active error for %s/%s: %s", repo, work_item_id, e) + return False + + def write_marker(repo: str, work_item_id: str | None, name: str, content: str = "") -> bool: """Create/overwrite a sentinel (best-effort). Returns True on success.""" try: diff --git a/src/stage_engine.py b/src/stage_engine.py index 36b169d..54f1582 100644 --- a/src/stage_engine.py +++ b/src/stage_engine.py @@ -384,6 +384,29 @@ def advance_stage( f"(auto-advance after {agent})" ) + # ORCH-021: arm post-deploy monitoring PAST `done`. Responsibility extends + # beyond the restart-time health-check to catch the "green deploy, red prod" + # class (ET-8). Idempotent (sentinel `armed`) + conditional (applies()), so a + # double webhook / reconciler / finalizer re-driving `done` never doubles it + # and non-applicable repos are untouched. never-raise (arm_monitor + guard). + # + # ORCH-094 (ADR-001 D3): the arm block is moved ABOVE the terminal-sync + # block (it used to run AFTER set_issue_monitoring). The order matters now + # that set_issue_monitoring is terminal-window-aware: by the time the + # legitimate first `Monitoring` is set, the task is ALREADY DB-`done` + # (update_task_stage ran above), so the guard must see the window as ACTIVE + # (ARMED & not DONE) to let it through. Arming first writes the ARMED + # sentinel -> window_active==True -> the guard returns ALLOW. A re-drive of + # deploy->done AFTER the window has closed (DONE present) -> window_active + # False -> the guard converges to Done (no resurrected Monitoring). The + # move is safe: arm_monitor only writes a sentinel + enqueues a deferred + # job; it depends on neither the Plane status nor the merge lease. + if next_stage == "done" and post_deploy.post_deploy_applies(repo): + try: + post_deploy.arm_monitor(repo, work_item_id, branch, task_id) + except Exception as e: # noqa: BLE001 - monitoring must never crash done + logger.warning(f"Task {task_id}: post-deploy arm failed: {e}") + # --- Terminal sync: deploy -> done must reach Plane's Done ----------- # When the deployer's check_deploy_status passes we advance to the # terminal 'done' stage. Previously a merged-PR webhook completed the @@ -401,7 +424,7 @@ def advance_stage( if next_stage == "done" and work_item_id: try: if post_deploy.post_deploy_applies(repo): - set_issue_monitoring(work_item_id) + set_issue_monitoring(work_item_id, reason="advance:deploy->done") logger.info( f"Task {task_id}: deploy->done (self), Plane state -> " f"Monitoring after Deploy (post-deploy window)" @@ -416,24 +439,14 @@ def advance_stage( # ORCH-043: the merge has landed (deploy->done). Release the merge lease as # a backstop in case the PR-merged webhook was lost (holder-aware no-op if a - # different task already owns it). Never raises. + # different task already owns it). Never raises. ORCH-094: stays AFTER the + # terminal-sync (the arm-block move above does not touch the lease). if next_stage == "done": try: merge_gate.release_merge_lease(repo, branch) except Exception as e: # noqa: BLE001 - defensive logger.warning(f"Task {task_id}: merge-lease release on done failed: {e}") - # ORCH-021: arm post-deploy monitoring PAST `done`. Responsibility extends - # beyond the restart-time health-check to catch the "green deploy, red prod" - # class (ET-8). Idempotent (sentinel `armed`) + conditional (applies()), so a - # double webhook / reconciler / finalizer re-driving `done` never doubles it - # and non-applicable repos are untouched. never-raise (arm_monitor + guard). - if next_stage == "done" and post_deploy.post_deploy_applies(repo): - try: - post_deploy.arm_monitor(repo, work_item_id, branch, task_id) - except Exception as e: # noqa: BLE001 - monitoring must never crash done - logger.warning(f"Task {task_id}: post-deploy arm failed: {e}") - # --- Launch the next agent (ORCH-4 fix: current_stage, not next) ----- next_agent = get_agent_for_stage(current_stage) if next_agent: @@ -1214,8 +1227,8 @@ def _handle_self_deploy_phase_a( # ORCH-066 (AC-6/AC-13): Phase A approval-pending is now `Awaiting Deploy`, # which discharges `In Review` of the deploy-approval meaning (In Review # stays for analyst BRD/review approve-pending only). Degrades to In Review - # where the status is not created. - set_issue_awaiting_deploy(work_item_id) + # where the status is not created. ORCH-094: reason tags the caller (FR-4). + set_issue_awaiting_deploy(work_item_id, reason="phase_a") # ORCH-036: belt-and-suspenders — wipe any STALE deploy-state markers before # arming a fresh approve. A prior FAILED pass clears on rollback, but clearing # here too guarantees the entry to every new prod-deploy pass starts clean @@ -1312,8 +1325,9 @@ def _handle_self_deploy_phase_b(task_id, repo, work_item_id, branch, result: Adv ) # ORCH-066 (AC-7): the prod deploy is now in flight -> indicate `Deploying` # (degrades to In Progress where the status is not created). + # ORCH-094: reason tags the caller (FR-4). if work_item_id: - set_issue_deploying(work_item_id) + set_issue_deploying(work_item_id, reason="phase_b") task_desc = ( f"Work item: {work_item_id}\nRepo: {repo}\nBranch: {branch}\n" f"Stage: deploy\nNote: deploy-finalize poll (prod self-deploy initiated)." @@ -1714,7 +1728,7 @@ def run_post_deploy_monitor(job: dict): try: conn = get_db() row = conn.execute( - "SELECT work_item_id, branch FROM tasks WHERE id=?", (task_id,) + "SELECT work_item_id, branch, stage FROM tasks WHERE id=?", (task_id,) ).fetchone() conn.close() except Exception as e: # noqa: BLE001 - never-raise @@ -1723,13 +1737,28 @@ def run_post_deploy_monitor(job: dict): if not row: logger.error(f"post-deploy-monitor: no task row for task_id={task_id}") return - work_item_id, branch = row[0], row[1] + work_item_id, branch, db_stage = row[0], row[1], row[2] # AC-15: a finished window is a no-op (defends against a duplicate job). if post_deploy.has_marker(repo, work_item_id, post_deploy.DONE): logger.info(f"post-deploy-monitor: {work_item_id} already done (no-op)") return + # ORCH-094 (FR-3 / D4 / AC-3): a tick must have an active basis. If the task + # became terminal ANOMALOUSLY mid-window (cancelled via STOP, ORCH-090), the + # tick is a "zombie" — close the window WITHOUT a status PATCH and WITHOUT + # re-queueing the next tick (a cancelled task already reached its own terminal; + # stamping a deploy status over it would flapp). A `done` stage is the NORMAL + # state of a post-deploy window (it opens strictly past deploy->done) so it is + # NOT treated as an anomaly here. + if (db_stage or "").strip() == "cancelled": + logger.info( + f"post-deploy-monitor: {work_item_id} task cancelled mid-window -> " + f"closing window, no status PATCH, no re-queue (zombie-tick guard)" + ) + post_deploy.mark_done(repo, work_item_id) + return + # One probe -> append -> classify (restart-safe via the persisted series). probe = post_deploy.probe_signals(settings.post_deploy_base_url) series = post_deploy.append_probe(repo, work_item_id, probe) diff --git a/tests/test_config.py b/tests/test_config.py index 4e6da8d..581f0f3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -292,3 +292,30 @@ def test_merge_retry_settings_env_override(monkeypatch): assert s.merge_retry_max_attempts == 5 assert s.merge_retry_backoff_base_s == 1 assert s.merge_retry_backoff_max_s == 8 + + +# --------------------------------------------------------------------------- +# ORCH-094: deploy_status_guard_* settings defaults + env override. +# --------------------------------------------------------------------------- +_DEPLOY_GUARD_ENV = ( + "ORCH_DEPLOY_STATUS_GUARD_ENABLED", + "ORCH_DEPLOY_STATUS_GUARD_REPOS", +) + + +def test_deploy_status_guard_settings_defaults(monkeypatch): + """Documented defaults: enabled True, repos empty (self-hosting only).""" + for name in _DEPLOY_GUARD_ENV: + monkeypatch.delenv(name, raising=False) + s = Settings() + assert s.deploy_status_guard_enabled is True + assert s.deploy_status_guard_repos == "" + + +def test_deploy_status_guard_settings_env_override(monkeypatch): + """Each field is read from its ORCH_DEPLOY_STATUS_GUARD_* env var.""" + monkeypatch.setenv("ORCH_DEPLOY_STATUS_GUARD_ENABLED", "false") + monkeypatch.setenv("ORCH_DEPLOY_STATUS_GUARD_REPOS", "orchestrator,enduro-trails") + s = Settings() + assert s.deploy_status_guard_enabled is False + assert s.deploy_status_guard_repos == "orchestrator,enduro-trails" diff --git a/tests/test_deploy_approve.py b/tests/test_deploy_approve.py index 256c876..1eae393 100644 --- a/tests/test_deploy_approve.py +++ b/tests/test_deploy_approve.py @@ -132,7 +132,8 @@ def test_tc05_no_approve_does_not_call_prod_hook(monkeypatch): # The restart-safe approve-requested marker was written. assert self_deploy.has_marker("orchestrator", "ORCH-036", self_deploy.APPROVE_REQUESTED) # ORCH-066 AC-6/AC-13: Phase A indicates `Awaiting Deploy`, NOT `In Review`. - stage_engine.set_issue_awaiting_deploy.assert_called_once_with("ORCH-036") + # ORCH-094: the caller now tags the reason (FR-4 observability). + stage_engine.set_issue_awaiting_deploy.assert_called_once_with("ORCH-036", reason="phase_a") stage_engine.set_issue_in_review.assert_not_called() @@ -161,7 +162,8 @@ def test_tc06_approved_calls_prod_hook_exactly_once(monkeypatch): assert any(j["agent"] == "deploy-finalizer" for j in _jobs()) assert self_deploy.has_marker("orchestrator", "ORCH-036", self_deploy.INITIATED) # ORCH-066 AC-7: Phase B indicates `Deploying` on a successful initiate. - stage_engine.set_issue_deploying.assert_called_once_with("ORCH-036") + # ORCH-094: the caller now tags the reason (FR-4 observability). + stage_engine.set_issue_deploying.assert_called_once_with("ORCH-036", reason="phase_b") # 2nd (duplicate) Confirm Deploy -> idempotent no-op, hook NOT called again. res2 = advance_stage( diff --git a/tests/test_deploy_status_observability.py b/tests/test_deploy_status_observability.py new file mode 100644 index 0000000..7c8c620 --- /dev/null +++ b/tests/test_deploy_status_observability.py @@ -0,0 +1,88 @@ +"""ORCH-094 — observability of deploy-status setting (FR-4 / AC-5 / TC-09). + +Every deploy-phase status decision emits ONE structured line carrying work_item, +caller (reason), target_status, db_stage, window_active and the verdict; a +suppression/convergence is logged explicitly so a future flapp is attributable. +""" +import logging +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_deploy_status_obs.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import deploy_status_guard as guard # noqa: E402 +from src import post_deploy # noqa: E402 +from src import config as cfg # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False) + monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False) + monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path)) + yield + + +def _make_task(stage, repo="orchestrator", wi="ORCH-061", branch="feature/ORCH-061-x"): + conn = get_db() + conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + conn.commit() + conn.close() + + +def test_tc09_converge_logs_full_attribution(caplog): + _make_task("done") + with caplog.at_level(logging.INFO, logger="orchestrator.deploy_status_guard"): + verdict = guard.decide("ORCH-061", guard.MONITORING, reason="advance:deploy->done") + assert verdict == guard.CONVERGE_DONE + rec = [r for r in caplog.records if r.name == "orchestrator.deploy_status_guard"] + assert rec, "guard emitted no observability record" + msg = rec[-1].getMessage() + # All five attribution fields + verdict are present. + for token in ( + "work_item=ORCH-061", "caller=advance:deploy->done", "target=monitoring", + "db_stage=done", "window_active=False", "verdict=CONVERGE_DONE", + ): + assert token in msg, f"missing {token!r} in {msg!r}" + # A convergence is logged at WARNING (easy to grep on a future flapp). + assert rec[-1].levelno == logging.WARNING + + +def test_tc09_allow_active_window_logged(caplog): + _make_task("done") + post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed") + with caplog.at_level(logging.INFO, logger="orchestrator.deploy_status_guard"): + verdict = guard.decide("ORCH-061", guard.MONITORING, reason="advance:deploy->done") + assert verdict == guard.ALLOW + rec = [r for r in caplog.records if r.name == "orchestrator.deploy_status_guard"][-1] + msg = rec.getMessage() + assert "window_active=True" in msg and "verdict=ALLOW" in msg + assert rec.levelno == logging.INFO + + +def test_tc09_suppress_cancelled_logged(caplog): + _make_task("cancelled") + with caplog.at_level(logging.INFO, logger="orchestrator.deploy_status_guard"): + verdict = guard.decide("ORCH-061", guard.AWAITING, reason="phase_a") + assert verdict == guard.SUPPRESS + rec = [r for r in caplog.records if r.name == "orchestrator.deploy_status_guard"][-1] + assert "verdict=SUPPRESS" in rec.getMessage() + assert "db_stage=cancelled" in rec.getMessage() + assert rec.levelno == logging.WARNING diff --git a/tests/test_deploy_status_terminal_guard.py b/tests/test_deploy_status_terminal_guard.py new file mode 100644 index 0000000..0633ec3 --- /dev/null +++ b/tests/test_deploy_status_terminal_guard.py @@ -0,0 +1,217 @@ +"""ORCH-094 — terminal-window-aware deploy-status guard (FR-2 / FR-5). + +Covers (04-test-plan.yaml): + TC-01 deploy-status for a DB stage=done task converges to Done: a + set_issue_monitoring/awaiting/deploying attempt on a terminal task drives + Done (or no-op if already Done), never an intermediate status. + TC-02 idempotency: a repeated terminal-aware setter call on an already-Done task + never PATCHes an intermediate status (no Done<->deploy pendulum). + TC-03 a non-terminal task (stage=deploy) is NOT suppressed: the deploy setters + proceed normally (regression AC-4). + TC-04 kill-switch off -> 1:1 prior behaviour (guard inert); on -> converge. + TC-05 never-raise: an undeterminable DB stage / DB error degrades safely (ALLOW, + no flapp, no exception). + TC-12 non-self repo: zero regression — the guard is inert (self-hosting only). +""" +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_deploy_status_guard.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import deploy_status_guard as guard # noqa: E402 +from src import plane_sync # noqa: E402 +from src import post_deploy # noqa: E402 +from src import config as cfg # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + # Guard ON, self-hosting only (empty CSV) by default. + monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False) + monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False) + # post-deploy sentinels live under a fresh tmp dir (window closed by default). + monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path)) + yield + + +def _make_task(stage, repo="orchestrator", wi="ORCH-061", branch="feature/ORCH-061-x"): + conn = get_db() + conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + conn.commit() + conn.close() + + +@pytest.fixture +def spy_setters(monkeypatch): + """Spy the low-level PATCH primitive + the Done convergence target.""" + direct = MagicMock() + done = MagicMock() + monkeypatch.setattr(plane_sync, "_set_issue_state_direct", direct) + monkeypatch.setattr(plane_sync, "set_issue_done", done) + # Keep status resolution offline-deterministic. + monkeypatch.setattr(plane_sync, "_resolve_project_id", lambda w=None, p=None: "proj-1") + monkeypatch.setattr( + plane_sync, "get_project_states", + lambda pid: {"awaiting_deploy": "S-aw", "deploying": "S-dep", "monitoring": "S-mon"}, + ) + return direct, done + + +# --- TC-01 ------------------------------------------------------------------ +def test_tc01_done_task_converges_to_done(spy_setters): + direct, done = spy_setters + _make_task("done") + # Window is NOT active (no ARMED sentinel) -> Monitoring is spurious. + for setter in ( + plane_sync.set_issue_monitoring, + plane_sync.set_issue_awaiting_deploy, + plane_sync.set_issue_deploying, + ): + done.reset_mock() + direct.reset_mock() + setter("ORCH-061") + # Converged to Done; no intermediate deploy-status PATCH. + done.assert_called_once_with("ORCH-061") + direct.assert_not_called() + + +def test_tc01_decide_verdicts_for_done(): + _make_task("done") + # No window -> all three converge. + assert guard.decide("ORCH-061", guard.MONITORING) == guard.CONVERGE_DONE + assert guard.decide("ORCH-061", guard.AWAITING) == guard.CONVERGE_DONE + assert guard.decide("ORCH-061", guard.DEPLOYING) == guard.CONVERGE_DONE + + +def test_tc01_decide_allows_monitoring_in_active_window(tmp_path, monkeypatch): + _make_task("done") + # Arm the window: ARMED present, DONE absent. + post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed") + assert post_deploy.window_active("orchestrator", "ORCH-061") is True + assert guard.decide("ORCH-061", guard.MONITORING) == guard.ALLOW + # Awaiting/Deploying are ALWAYS spurious for a done task, even with a window. + assert guard.decide("ORCH-061", guard.AWAITING) == guard.CONVERGE_DONE + # Once the window closes (DONE present) Monitoring converges too. + post_deploy.mark_done("orchestrator", "ORCH-061") + assert post_deploy.window_active("orchestrator", "ORCH-061") is False + assert guard.decide("ORCH-061", guard.MONITORING) == guard.CONVERGE_DONE + + +# --- TC-02 ------------------------------------------------------------------ +def test_tc02_idempotent_no_pendulum(spy_setters): + direct, done = spy_setters + _make_task("done") + # Repeated calls keep converging to Done; the intermediate Monitoring PATCH + # never fires, so there is no Done<->deploy-status pendulum. + for _ in range(5): + plane_sync.set_issue_monitoring("ORCH-061") + assert direct.call_count == 0 + assert done.call_count == 5 # idempotent PATCH-equivalent (same terminal state) + + +# --- TC-03 ------------------------------------------------------------------ +def test_tc03_non_terminal_not_suppressed(spy_setters): + direct, done = spy_setters + _make_task("deploy") # a really-deploying task + plane_sync.set_issue_awaiting_deploy("ORCH-061") + plane_sync.set_issue_deploying("ORCH-061") + plane_sync.set_issue_monitoring("ORCH-061") + # All three proceed to a real PATCH; nothing converges to Done. + assert direct.call_count == 3 + done.assert_not_called() + assert guard.decide("ORCH-061", guard.MONITORING) == guard.ALLOW + + +# --- TC-04 ------------------------------------------------------------------ +def test_tc04_kill_switch(spy_setters, monkeypatch): + direct, done = spy_setters + _make_task("done") + # OFF -> terminal-blind, the monitoring PATCH proceeds (1:1 pre-ORCH-094). + monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", False) + plane_sync.set_issue_monitoring("ORCH-061") + assert direct.call_count == 1 + done.assert_not_called() + # ON -> converge to Done. + monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True) + direct.reset_mock() + done.reset_mock() + plane_sync.set_issue_monitoring("ORCH-061") + direct.assert_not_called() + done.assert_called_once_with("ORCH-061") + + +# --- TC-05 ------------------------------------------------------------------ +def test_tc05_never_raise_on_db_error(spy_setters, monkeypatch): + direct, done = spy_setters + _make_task("done") + + def _boom(_wi): + raise RuntimeError("db down") + + monkeypatch.setattr(_db, "get_task_by_work_item_id", _boom) + # decide degrades to ALLOW (fail-safe), never raises. + assert guard.decide("ORCH-061", guard.MONITORING) == guard.ALLOW + # The setter proceeds with the normal PATCH (1:1), no convergence, no crash. + plane_sync.set_issue_monitoring("ORCH-061") + assert direct.call_count == 1 + done.assert_not_called() + + +def test_tc05_unknown_task_allows(spy_setters): + direct, done = spy_setters + # No task row at all -> ALLOW (foreign/unknown issue, not ours). + assert guard.decide("ORCH-999", guard.MONITORING) == guard.ALLOW + plane_sync.set_issue_monitoring("ORCH-999") + assert direct.call_count == 1 + done.assert_not_called() + + +def test_tc05_cancelled_is_suppressed(spy_setters): + direct, done = spy_setters + _make_task("cancelled") + assert guard.decide("ORCH-061", guard.MONITORING) == guard.SUPPRESS + plane_sync.set_issue_monitoring("ORCH-061") + # Suppressed: neither an intermediate PATCH nor a Done convergence. + direct.assert_not_called() + done.assert_not_called() + + +# --- TC-12 ------------------------------------------------------------------ +def test_tc12_non_self_repo_inert(spy_setters): + direct, done = spy_setters + # A non-self repo done task: the guard is inert (self-hosting only, empty CSV). + _make_task("done", repo="enduro-trails", wi="ET-042", branch="feature/ET-042-x") + assert guard.applies("enduro-trails") is False + assert guard.decide("ET-042", guard.MONITORING) == guard.ALLOW + plane_sync.set_issue_monitoring("ET-042") + # Behaviour unchanged: the requested PATCH proceeds, no convergence. + assert direct.call_count == 1 + done.assert_not_called() + + +def test_tc12_csv_scope_overrides_self_hosting(monkeypatch): + _make_task("done", repo="enduro-trails", wi="ET-042", branch="feature/ET-042-x") + # Explicit CSV scope brings a non-self repo in-scope. + monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "enduro-trails") + assert guard.applies("enduro-trails") is True + assert guard.applies("orchestrator") is False # not listed -> out of scope + assert guard.decide("ET-042", guard.MONITORING) == guard.CONVERGE_DONE diff --git a/tests/test_deploy_terminal_sync.py b/tests/test_deploy_terminal_sync.py index c417cef..725db41 100644 --- a/tests/test_deploy_terminal_sync.py +++ b/tests/test_deploy_terminal_sync.py @@ -135,7 +135,10 @@ def test_tc08_self_deploy_done_sets_monitoring_not_done(monkeypatch): assert _stage(task_id) == "done" # Self-hosting: the issue enters the Monitoring window, NOT terminal Done yet. - stage_engine.set_issue_monitoring.assert_called_once_with("ORCH-036") + # ORCH-094: the terminal-sync caller now tags the reason (FR-4 observability). + stage_engine.set_issue_monitoring.assert_called_once_with( + "ORCH-036", reason="advance:deploy->done" + ) stage_engine.set_issue_done.assert_not_called() diff --git a/tests/test_post_deploy_monitor_termination.py b/tests/test_post_deploy_monitor_termination.py new file mode 100644 index 0000000..46a1113 --- /dev/null +++ b/tests/test_post_deploy_monitor_termination.py @@ -0,0 +1,170 @@ +"""ORCH-094 — deterministic post-deploy-monitor termination (FR-3 / AC-3). + +Covers (04-test-plan.yaml): + TC-06 after the window finishes (HEALTHY, ticks==budget -> set_issue_done + + `done` marker) there are NO further status PATCHes for the task (a second + tick is a no-op: 0 set_issue_* calls). + TC-07 a tick at DB stage=done with a closed window OR a task cancelled mid-window + -> immediate no-op: no status PATCH and no next-tick enqueue (zombie-tick + excluded). + TC-08 arm_monitor does not re-arm a task already in done (armed/done marker -> + no-op), and a deploy->done re-drive after the window closed converges to + Done instead of resurrecting Monitoring. +""" +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_post_deploy_termination.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import stage_engine # noqa: E402 +from src import post_deploy # noqa: E402 +from src import config as cfg # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path)) + # Small window so the budget is 1 tick (window // interval). + monkeypatch.setattr(stage_engine.settings, "post_deploy_window_s", 10) + monkeypatch.setattr(stage_engine.settings, "post_deploy_interval_s", 10) + monkeypatch.setattr(post_deploy.settings, "post_deploy_window_s", 10) + monkeypatch.setattr(post_deploy.settings, "post_deploy_interval_s", 10) + # write_post_deploy_log touches a worktree/git; stub it. + monkeypatch.setattr(post_deploy, "write_post_deploy_log", MagicMock(return_value=True)) + yield + + +@pytest.fixture +def spy_status(monkeypatch): + setters = {} + for name in ("set_issue_done", "set_issue_monitoring", "set_issue_awaiting_deploy", + "set_issue_deploying", "set_issue_blocked"): + m = MagicMock() + monkeypatch.setattr(stage_engine, name, m) + setters[name] = m + monkeypatch.setattr(stage_engine, "_notify_post_deploy", MagicMock()) + return setters + + +def _make_task(stage="done", repo="orchestrator", wi="ORCH-061", branch="feature/ORCH-061-x"): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + tid = cur.lastrowid + conn.commit() + conn.close() + return tid + + +def _jobs(): + conn = get_db() + rows = conn.execute("SELECT agent FROM jobs ORDER BY id").fetchall() + conn.close() + return [r[0] for r in rows] + + +def _healthy(*a, **k): + return post_deploy.ProbeResult(health_ok=True, total=2, fivexx=0, detail="ok") + + +# --- TC-06 ------------------------------------------------------------------ +def test_tc06_clean_finish_then_no_more_patches(spy_status, monkeypatch): + monkeypatch.setattr(post_deploy, "probe_signals", _healthy) + tid = _make_task("done") + post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed") + + job = {"task_id": tid, "repo": "orchestrator", "id": 1, "agent": "post-deploy-monitor"} + # Tick 1: budget==1, ticks==1 -> HEALTHY window exhausted -> finish. + stage_engine.run_post_deploy_monitor(job) + spy_status["set_issue_done"].assert_called_once_with("ORCH-061") + assert post_deploy.has_marker("orchestrator", "ORCH-061", post_deploy.DONE) + # No next tick was enqueued (window exhausted). + assert _jobs() == [] + + # Tick 2 (e.g. duplicate job): DONE marker present -> no-op, ZERO new PATCHes. + spy_status["set_issue_done"].reset_mock() + stage_engine.run_post_deploy_monitor(job) + spy_status["set_issue_done"].assert_not_called() + spy_status["set_issue_monitoring"].assert_not_called() + assert _jobs() == [] + + +# --- TC-07 ------------------------------------------------------------------ +def test_tc07_cancelled_mid_window_is_noop(spy_status, monkeypatch): + monkeypatch.setattr(post_deploy, "probe_signals", _healthy) + tid = _make_task("cancelled") + post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed") + + job = {"task_id": tid, "repo": "orchestrator", "id": 1, "agent": "post-deploy-monitor"} + stage_engine.run_post_deploy_monitor(job) + # Zombie-tick guard: window closed, NO status PATCH, NO next tick. + for name, m in spy_status.items(): + m.assert_not_called() + assert post_deploy.has_marker("orchestrator", "ORCH-061", post_deploy.DONE) + assert _jobs() == [] + + +def test_tc07_finished_window_is_noop(spy_status, monkeypatch): + monkeypatch.setattr(post_deploy, "probe_signals", _healthy) + tid = _make_task("done") + # Window already finished (DONE marker present) -> no active basis to tick. + post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed") + post_deploy.mark_done("orchestrator", "ORCH-061") + + job = {"task_id": tid, "repo": "orchestrator", "id": 1, "agent": "post-deploy-monitor"} + stage_engine.run_post_deploy_monitor(job) + spy_status["set_issue_done"].assert_not_called() + spy_status["set_issue_monitoring"].assert_not_called() + assert _jobs() == [] + + +# --- TC-08 ------------------------------------------------------------------ +def test_tc08_arm_monitor_idempotent_no_rearm(monkeypatch): + tid = _make_task("done") + # First arm: writes ARMED + enqueues tick 1. + assert post_deploy.arm_monitor("orchestrator", "ORCH-061", "feature/ORCH-061-x", tid) is True + assert _jobs() == ["post-deploy-monitor"] + # Second arm (re-drive deploy->done): ARMED present -> no-op, no new job. + assert post_deploy.arm_monitor("orchestrator", "ORCH-061", "feature/ORCH-061-x", tid) is False + assert _jobs() == ["post-deploy-monitor"] + + +def test_tc08_redrive_after_window_closed_converges(spy_status, monkeypatch): + # Guard ON, self-hosting. + monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False) + monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False) + _make_task("done") + # Window armed then closed (a completed post-deploy observation). + post_deploy.write_marker("orchestrator", "ORCH-061", post_deploy.ARMED, "armed") + post_deploy.mark_done("orchestrator", "ORCH-061") + # A stale re-drive calling the REAL guarded setter must converge to Done, not + # resurrect Monitoring. (Use the real plane_sync setter via stage_engine import.) + from src import plane_sync + direct = MagicMock() + done = MagicMock() + monkeypatch.setattr(plane_sync, "_set_issue_state_direct", direct) + monkeypatch.setattr(plane_sync, "set_issue_done", done) + monkeypatch.setattr(plane_sync, "_resolve_project_id", lambda w=None, p=None: "proj-1") + monkeypatch.setattr(plane_sync, "get_project_states", lambda pid: {"monitoring": "S-mon"}) + + plane_sync.set_issue_monitoring("ORCH-061", reason="advance:deploy->done") + direct.assert_not_called() + done.assert_called_once_with("ORCH-061") diff --git a/tests/test_reconciler_done_deploy_convergence.py b/tests/test_reconciler_done_deploy_convergence.py new file mode 100644 index 0000000..a3057f0 --- /dev/null +++ b/tests/test_reconciler_done_deploy_convergence.py @@ -0,0 +1,82 @@ +"""ORCH-094 — sync convergence for a done task stuck on a deploy status (TC-10). + +Integration-level: ANY sync source (reconciler tick / monitor tick / a direct +deploy-status setter call) that touches a DB-done task converges Plane to Done +idempotently instead of an intermediate deploy status, and a repeated tick does +NOT swing the Done<->deploy-status pendulum. The guard lives on the setter +(ADR-001 D1/D7), so the reconciler code itself is unchanged — driving the setter +the way a stale actor would is the faithful reproduction of the 061 flapp. +""" +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_reconciler_done_converge.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import plane_sync # noqa: E402 +from src import post_deploy # noqa: E402 +from src import config as cfg # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False) + monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False) + monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path)) + yield + + +@pytest.fixture +def spy(monkeypatch): + direct = MagicMock() + done = MagicMock() + monkeypatch.setattr(plane_sync, "_set_issue_state_direct", direct) + monkeypatch.setattr(plane_sync, "set_issue_done", done) + monkeypatch.setattr(plane_sync, "_resolve_project_id", lambda w=None, p=None: "proj-1") + monkeypatch.setattr( + plane_sync, "get_project_states", + lambda pid: {"awaiting_deploy": "S-aw", "deploying": "S-dep", "monitoring": "S-mon"}, + ) + return direct, done + + +def _make_task(stage="done", repo="orchestrator", wi="ORCH-061"): + conn = get_db() + conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, "feature/ORCH-061-x", stage), + ) + conn.commit() + conn.close() + + +def test_tc10_repeated_sync_converges_no_pendulum(spy): + direct, done = spy + _make_task("done") # done, window closed (no ARMED sentinel) + # Simulate many sync ticks alternately trying to set Monitoring / Awaiting, + # exactly like the observed 061 pendulum (Awaiting <-> Monitoring forever). + for i in range(10): + if i % 2 == 0: + plane_sync.set_issue_monitoring("ORCH-061", reason="reconciler-tick") + else: + plane_sync.set_issue_awaiting_deploy("ORCH-061", reason="reconciler-tick") + # Every tick converged to Done; not a single intermediate deploy-status PATCH. + assert direct.call_count == 0 + assert done.call_count == 10 + # All convergence calls target the same terminal Done (no swing). + assert all(c.args == ("ORCH-061",) for c in done.call_args_list) diff --git a/tests/test_self_deploy_cycle_regression.py b/tests/test_self_deploy_cycle_regression.py new file mode 100644 index 0000000..fda6cfd --- /dev/null +++ b/tests/test_self_deploy_cycle_regression.py @@ -0,0 +1,128 @@ +"""ORCH-094 — the real deploy cycle is NOT suppressed by the guard (TC-11 / AC-4). + +A genuinely-deploying (non-terminal) self-hosting task must still walk +`Awaiting Deploy -> Deploying -> Monitoring after Deploy -> Done` exactly as before +ORCH-094. The critical regression case is the LEGITIMATE first `Monitoring`: by the +time the terminal-sync runs the task is ALREADY DB-`done` (update_task_stage ran +above), so the guard would wrongly converge it to Done UNLESS the arm-block moved +ABOVE the terminal-sync (ADR-001 D3) marks the post-deploy window active first. +This test exercises that ordering end-to-end via run_deploy_finalizer with the REAL +guard + REAL arm_monitor wired in (only the network PATCH primitive is mocked). +""" +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_self_deploy_cycle_regression.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import stage_engine # noqa: E402 +from src import plane_sync # noqa: E402 +from src import post_deploy # noqa: E402 +from src import self_deploy # noqa: E402 +from src import config as cfg # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr(self_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(self_deploy.settings, "host_repos_dir", str(tmp_path)) + monkeypatch.setattr(post_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(post_deploy.settings, "host_repos_dir", str(tmp_path)) + # Guard ON, self-hosting only. + monkeypatch.setattr(cfg.settings, "deploy_status_guard_enabled", True, raising=False) + monkeypatch.setattr(cfg.settings, "deploy_status_guard_repos", "", raising=False) + # Post-deploy monitor applies for self repo (arm fires on deploy->done). + monkeypatch.setattr(post_deploy.settings, "post_deploy_monitor_enabled", True) + monkeypatch.setattr(post_deploy.settings, "post_deploy_repos", "") + monkeypatch.setattr(stage_engine.post_deploy.settings, "post_deploy_monitor_enabled", True) + monkeypatch.setattr(stage_engine.post_deploy.settings, "post_deploy_repos", "") + # Stub the worktree/git artefact writers. + monkeypatch.setattr(stage_engine.self_deploy, "write_deploy_log", MagicMock(return_value=True)) + monkeypatch.setattr(stage_engine.merge_gate, "release_merge_lease", MagicMock()) + yield + + +@pytest.fixture +def spy_plane(monkeypatch): + """Spy plane_sync's low-level PATCH + Done convergence (the REAL guard runs).""" + direct = MagicMock() + done = MagicMock() + monkeypatch.setattr(plane_sync, "_set_issue_state_direct", direct) + monkeypatch.setattr(plane_sync, "set_issue_done", done) + monkeypatch.setattr(plane_sync, "_resolve_project_id", lambda w=None, p=None: "proj-1") + monkeypatch.setattr( + plane_sync, "get_project_states", + lambda pid: {"awaiting_deploy": "S-aw", "deploying": "S-dep", "monitoring": "S-mon", + "done": "S-done"}, + ) + # stage_engine.set_issue_done is a module-level binding -> patch it too so a + # non-self / fallback Done path is observable; here we expect Monitoring though. + monkeypatch.setattr(stage_engine, "set_issue_done", done) + return direct, done + + +def _make_task(stage, repo="orchestrator", wi="ORCH-063", branch="feature/ORCH-063-x"): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + tid = cur.lastrowid + conn.commit() + conn.close() + return tid + + +def _pass(*a, **k): + return (True, "ok") + + +def test_tc11_non_terminal_awaiting_deploying_pass(spy_plane): + direct, done = spy_plane + _make_task("deploy") + # Phase A / Phase B statuses on a NON-terminal task proceed (no convergence). + plane_sync.set_issue_awaiting_deploy("ORCH-063", reason="phase_a") + plane_sync.set_issue_deploying("ORCH-063", reason="phase_b") + assert direct.call_count == 2 + done.assert_not_called() + + +def test_tc11_legit_monitoring_preserved_on_deploy_done(spy_plane, monkeypatch): + direct, done = spy_plane + # Hook reported exit 0. + self_deploy.write_marker("orchestrator", "ORCH-063", self_deploy.RESULT, "0") + monkeypatch.setattr( + stage_engine, "QG_CHECKS", + {**stage_engine.QG_CHECKS, "check_deploy_status": _pass}, + ) + + tid = _make_task("deploy") + stage_engine.run_deploy_finalizer( + {"task_id": tid, "repo": "orchestrator", "id": 1, "agent": "deploy-finalizer"} + ) + + # Stage advanced to done. + conn = get_db() + stage = conn.execute("SELECT stage FROM tasks WHERE id=?", (tid,)).fetchone()[0] + conn.close() + assert stage == "done" + # The arm-block ran BEFORE terminal-sync -> the window is active -> the guard + # ALLOWS the legitimate Monitoring PATCH (S-mon), it is NOT converged to Done. + assert post_deploy.has_marker("orchestrator", "ORCH-063", post_deploy.ARMED) + mon_calls = [c for c in direct.call_args_list if c.args[1] == "S-mon"] + assert len(mon_calls) == 1, f"expected one Monitoring PATCH, got {direct.call_args_list}" + done.assert_not_called()