From 63187ff10231331783de2c03d2829213284849d9 Mon Sep 17 00:00:00 2001 From: claude-bot Date: Sat, 6 Jun 2026 20:01:07 +0000 Subject: [PATCH] developer(ET): auto-commit from developer run_id=192 --- docs/operations/DEPLOY_HOOK.md | 16 ++ docs/operations/INFRA.md | 9 +- src/agents/launcher.py | 28 +++ src/config.py | 44 ++++ src/self_deploy.py | 312 +++++++++++++++++++++++++ src/stage_engine.py | 231 ++++++++++++++++++ tests/test_deploy_approve.py | 160 +++++++++++++ tests/test_deploy_build_once.py | 47 ++++ tests/test_deploy_hook_mapping.py | 47 ++++ tests/test_deploy_hook_rollback_sim.py | 118 ++++++++++ tests/test_deploy_notifications.py | 102 ++++++++ tests/test_deploy_rollback.py | 100 ++++++++ tests/test_deploy_routing.py | 174 ++++++++++++++ tests/test_deploy_terminal_sync.py | 104 +++++++++ tests/test_qg_checks.py | 53 +++++ tests/test_stage_engine.py | 7 +- tests/test_stages.py | 41 ++++ tests/test_staging_precondition.py | 99 ++++++++ 18 files changed, 1690 insertions(+), 2 deletions(-) create mode 100644 src/self_deploy.py create mode 100644 tests/test_deploy_approve.py create mode 100644 tests/test_deploy_build_once.py create mode 100644 tests/test_deploy_hook_mapping.py create mode 100644 tests/test_deploy_hook_rollback_sim.py create mode 100644 tests/test_deploy_notifications.py create mode 100644 tests/test_deploy_rollback.py create mode 100644 tests/test_deploy_routing.py create mode 100644 tests/test_deploy_terminal_sync.py create mode 100644 tests/test_qg_checks.py create mode 100644 tests/test_stages.py create mode 100644 tests/test_staging_precondition.py diff --git a/docs/operations/DEPLOY_HOOK.md b/docs/operations/DEPLOY_HOOK.md index ba3cafb..0f81102 100644 --- a/docs/operations/DEPLOY_HOOK.md +++ b/docs/operations/DEPLOY_HOOK.md @@ -8,6 +8,7 @@ 1. **Захват текущего образа** — до рестарта записывает ID образа работающего контейнера в `$PREV_IMAGE_FILE` (best-effort, не падает если сервис не запущен). 2. **git pull** — обновляет код репозитория. +2b. **Build-once retag** (ORCH-036, BR-6) — если задан `$SOURCE_IMAGE`, хук ретегает его на `$TARGET_IMAGE` (`docker tag $SOURCE_IMAGE $TARGET_IMAGE`) и поднимает контейнер на этом образе через `up -d --no-build`. Это деплой РОВНО того образа, что прошёл staging, **без `docker build`**. Если `$SOURCE_IMAGE` не задан (дефолт) — шаг пропускается (обратная совместимость). 3. **Рестарт контейнера** — `docker compose --profile $COMPOSE_PROFILE up -d --no-build $TARGET_SERVICE`. 4. **Health-цикл** — 10 попыток × 6с = до 60с. Критерий: HTTP 200 + тело содержит `"status":"ok"`. - **Успех** → `exit 0`, лог "Deploy SUCCESS". @@ -29,6 +30,7 @@ | `TARGET_IMAGE` | `orchestrator-orchestrator-staging` | Имя образа для retag при rollback | | `COMPOSE_PROFILE`| `staging` | Docker compose profile (пусто = без профиля) | | `PREV_IMAGE_FILE`| `$REPO/.deploy-prev-image-staging`| Файл для сохранения предыдущего образа | +| `SOURCE_IMAGE` | _(unset)_ | Build-once (ORCH-036): провалидированный образ для retag на `$TARGET_IMAGE` перед рестартом (без rebuild). Не задан → шаг пропущен. | | `LOG` | `/var/log/orchestrator/deploy-hook.log` | Лог-файл (fallback: `$REPO/deploy-hook.log`) | > ⚠️ **Дефолт — всегда STAGING**. Прод активируется только явным переопределением env. @@ -55,6 +57,20 @@ PREV_IMAGE_FILE=/home/slin/repos/orchestrator/.deploy-prev-image-prod \ bash scripts/orchestrator-deploy-hook.sh --deploy ``` +### Прод build-once (ORCH-036) — ретег staging-образа, без rebuild + +Так прод-деплой запускается **автоматически** исполняемым самодеплоем (Фаза B: `ssh + setsid`, см. `INFRA.md`). Ключевое отличие — `SOURCE_IMAGE` указывает на провалидированный staging-образ, который ретегается на прод-тег: + +```bash +SOURCE_IMAGE=orchestrator-orchestrator-staging \ +TARGET_SERVICE=orchestrator \ +TARGET_PORT=8500 \ +TARGET_IMAGE=orchestrator-orchestrator \ +COMPOSE_PROFILE="" \ +PREV_IMAGE_FILE=/home/slin/repos/orchestrator/.deploy-prev-image-prod \ +bash scripts/orchestrator-deploy-hook.sh --deploy +``` + ### Ручной rollback staging ```bash diff --git a/docs/operations/INFRA.md b/docs/operations/INFRA.md index 38901b5..bb8fbfd 100644 --- a/docs/operations/INFRA.md +++ b/docs/operations/INFRA.md @@ -75,7 +75,14 @@ ADR `docs/work-items/ORCH-040/06-adr/ADR-001-run-agents-as-host-uid.md` и гл | `ORCH_AGENT_EFFORT_DEFAULT` | режим работы `--effort` по умолчанию (ORCH-41): low\|medium\|high\|xhigh\|max; дефолт `high` | | `ORCH_AGENT_EFFORT_` | per-agent effort; дефолт: думающие → high, tester/deployer → medium | | `ORCH_AGENT_FALLBACK_MODEL` | опц. фолбэк-модель при overloaded (`--fallback-model`); пусто → без флага | -| `DEPLOY_SSH_USER` / `_HOST` / `DEPLOY_HOOK_SCRIPT` | параметры деплой-хука | +| `ORCH_SELF_DEPLOY_ENABLED` | ORCH-036 kill-switch исполняемого самодеплоя (true); false → legacy-путь для всех | +| `ORCH_SELF_DEPLOY_REPOS` | CSV репозиториев с реальным самодеплоем; пусто → только self-hosting `orchestrator` | +| `ORCH_DEPLOY_REQUIRE_MANUAL_APPROVE` | требовать человеческий Plane «Approved» для прод-деплоя (true, безопасно) | +| `ORCH_DEPLOY_FINALIZE_DELAY_S` / `_MAX_ATTEMPTS` | задержка и бюджет defer'ов finalizer'а (Фаза C; 90 / 10) | +| `ORCH_DEPLOY_SSH_USER` / `_SSH_HOST` | куда запускается detached хост-деплой (Фаза B, `ssh user@host`) | +| `ORCH_DEPLOY_HOOK_SCRIPT` / `_HOST_REPO_PATH` | путь к хук-скрипту (отн. репо) и чекаут orchestrator на хосте | +| `ORCH_DEPLOY_PROD_SOURCE_IMAGE` | staging-образ для build-once retag на прод-тег (без rebuild) | +| `ORCH_DEPLOY_PROD_TARGET_SERVICE` / `_TARGET_PORT` / `_TARGET_IMAGE` / `_COMPOSE_PROFILE` / `_PREV_IMAGE_FILE` | прод-цель хука + снапшот для авто-rollback | **Секреты — только в `.env` / `.env.staging` на хосте, в гит НЕ коммитятся.** Канон — `.env.example`, `.env.staging.example`. diff --git a/src/agents/launcher.py b/src/agents/launcher.py index 43d8019..9d7598b 100644 --- a/src/agents/launcher.py +++ b/src/agents/launcher.py @@ -214,7 +214,14 @@ class AgentLauncher: Same spawn path as launch(), but threads job['id'] through so the monitor can update the job's status (done / requeue / failed) and link jobs.run_id to the agent_runs row. Returns the agent_run_id. + + ORCH-036: the reserved-agent ``deploy-finalizer`` is a DETERMINISTIC + (no-LLM) job — intercept it BEFORE _spawn (which would raise + "Unknown agent", R-6) and run the deploy finalizer synchronously, driving + the jobs row status itself. Returns None (no agent_run row). """ + if job.get("agent") == "deploy-finalizer": + return self._run_deploy_finalizer_job(job) return self._spawn( job["agent"], job["repo"], @@ -223,6 +230,27 @@ class AgentLauncher: job_id=job["id"], ) + def _run_deploy_finalizer_job(self, job: dict): + """ORCH-036 Phase C: run the deterministic deploy finalizer for a job. + + Not an LLM spawn — there is no subprocess/monitor, so we mark the jobs row + done/failed here. Any error is contained (the finalizer never-raises, but + we guard anyway so a finalizer fault can't wedge the worker). + """ + from ..db import mark_job + from .. import stage_engine + try: + stage_engine.run_deploy_finalizer(job) + mark_job(job["id"], "done") + logger.info(f"deploy-finalizer job {job['id']} done") + except Exception as e: + logger.error(f"deploy-finalizer job {job['id']} failed: {e}") + try: + mark_job(job["id"], "failed", error=f"deploy-finalizer error: {e}") + except Exception: + pass + return None + def _spawn(self, agent: str, repo: str, task_content: str = None, task_id: int = None, job_id: int = None) -> int: """Shared spawn implementation for launch() and launch_job(). diff --git a/src/config.py b/src/config.py index eceafc1..e1eb7b4 100644 --- a/src/config.py +++ b/src/config.py @@ -152,6 +152,50 @@ class Settings(BaseSettings): merge_defer_delay_s: int = 60 merge_defer_max_attempts: int = 5 + # ORCH-036: executable self-deploy (deploy stage drives the host hook). + # The `deploy` stage for the self-hosting repo is turned into a REAL prod + # restart via a detached host process, gated by a manual approve. Three-phase + # design (ADR-001): A=approve-request, B=initiate (human Approved), C=finalizer + # maps the hook exit-code -> deploy_status. Non-self repos are unaffected. + # + # self_deploy_enabled -> global kill-switch; False -> no Phase A/B/C + # interception (the legacy synchronous deployer + # path runs for everyone, env ORCH_SELF_DEPLOY_ENABLED). + # self_deploy_repos -> CSV of repos where executable self-deploy is + # REAL; empty -> only the self-hosting repo + # (orchestrator). Mirrors merge_gate_repos. + # deploy_require_manual_approve -> require a human Approved before the prod + # restart (BR-5). Default true; NOT toggled in + # ORCH-36 (AC-12). false -> Phase A initiates + # immediately (structural branch, off by default). + # deploy_finalize_delay_s -> delay before the first finalize poll; must be + # > the hook health-loop (~60s) so the verdict + # usually exists on the first poll. + # deploy_finalize_max_attempts -> bounded finalize-defer budget (anti-livelock). + # ssh / hook target (detached prod restart; real values live on the host): + # deploy_ssh_user / deploy_ssh_host -> ssh target for the host hook (INFRA P-2). + # deploy_hook_script -> path to the hook ON THE HOST (relative to repo). + # deploy_host_repo_path -> orchestrator clone path on the host. + # prod overrides passed to the hook for build-once (retag staging image -> prod): + # deploy_prod_source_image -> image validated on staging (retagged, no rebuild). + # deploy_prod_target_service / _port / _image / _compose_profile -> prod profile. + # deploy_prod_prev_image_file -> prod prev-image snapshot (separate from staging). + self_deploy_enabled: bool = True + self_deploy_repos: str = "" + deploy_require_manual_approve: bool = True + deploy_finalize_delay_s: int = 90 + deploy_finalize_max_attempts: int = 10 + deploy_ssh_user: str = "slin" + deploy_ssh_host: str = "" + deploy_hook_script: str = "scripts/orchestrator-deploy-hook.sh" + deploy_host_repo_path: str = "/home/slin/repos/orchestrator" + deploy_prod_source_image: str = "orchestrator-orchestrator-staging" + deploy_prod_target_service: str = "orchestrator" + deploy_prod_target_port: int = 8500 + deploy_prod_target_image: str = "orchestrator-orchestrator" + deploy_prod_compose_profile: str = "" + deploy_prod_prev_image_file: str = ".deploy-prev-image-prod" + # Telegram notifications telegram_bot_token: str = "" telegram_chat_id: str = "" diff --git a/src/self_deploy.py b/src/self_deploy.py new file mode 100644 index 0000000..eba5de8 --- /dev/null +++ b/src/self_deploy.py @@ -0,0 +1,312 @@ +"""Executable self-deploy primitives (ORCH-036). + +The ``deploy`` stage for the self-hosting ``orchestrator`` repo is a REAL prod +restart, not a paper LLM verdict. Because the prod container (8500) runs the +worker/agent itself, the restart must be performed by an EXTERNAL host process +that survives the container dying (BR-2). The orchestration is split into three +deterministic phases (ADR-001), wired in ``stage_engine``: + + * Phase A — request approve on the ``deploy-staging -> deploy`` edge. + * Phase B — a human Plane ``Approved`` initiates the detached host deploy. + * Phase C — a deterministic finalizer maps the hook exit-code -> deploy_status. + +This module is a **leaf**: it imports only config / git_worktree (and lazily +``qg.checks.is_self_hosting_repo``), never ``stage_engine`` / ``launcher`` — the +orchestration that needs those lives in ``stage_engine``. Every public helper +honours a **never-raise** contract so a deploy-state hiccup can never crash the +stage engine. + +Restart-safe state lives in sentinel files under +``/.deploy-state-//`` (mirrors the merge-lease +pattern, ТЗ §4 — no DB migration), on the shared mount visible to BOTH the +container (reads markers) and the host (writes ``result``): + * ``approve-requested`` — Phase A done; + * ``initiated`` — Phase B started (idempotency-guard); + * ``result`` — the hook exit-code, written by the host WRAPPER + (``echo $? > result``), NOT by the hook itself. +""" + +import logging +import os +import shlex +import subprocess + +from .config import settings + +logger = logging.getLogger("orchestrator.self_deploy") + +# Sentinel marker filenames (see module docstring). +APPROVE_REQUESTED = "approve-requested" +INITIATED = "initiated" +RESULT = "result" + +# ssh launch is detached (returns immediately); keep a bounded timeout so a hung +# ssh handshake never wedges the caller. +_SSH_TIMEOUT = 30 +_GIT_TIMEOUT = 60 + + +# --------------------------------------------------------------------------- +# Conditionality +# --------------------------------------------------------------------------- +def self_deploy_applies(repo: str) -> bool: + """Whether executable self-deploy (Phase A/B/C) is REAL for this repo. + + Mirrors the ORCH-35 / ORCH-43 conditional rollout: + * ``self_deploy_enabled=False`` -> always False (global kill-switch); the + legacy synchronous deployer path runs for everyone. + * ``self_deploy_repos`` (CSV) non-empty -> real only for listed repos. + * empty CSV -> real ONLY for the self-hosting repo (``orchestrator``). + Never raises. + """ + try: + if not settings.self_deploy_enabled: + return False + raw = (settings.self_deploy_repos or "").strip() + if raw: + allowed = {r.strip().lower() for r in raw.split(",") if r.strip()} + return (repo or "").strip().lower() in allowed + # Lazy import keeps this module a leaf (avoids importing qg at module load). + from .qg.checks import is_self_hosting_repo + return is_self_hosting_repo(repo) + except Exception as e: # noqa: BLE001 - never-raise contract + logger.warning("self_deploy_applies error for %s: %s", repo, e) + return False + + +# --------------------------------------------------------------------------- +# exit-code -> deploy_status mapping (pure, unit-tested: TC-01/02/03) +# --------------------------------------------------------------------------- +def map_exit_code_to_status(exit_code) -> str: + """Map a deploy-hook exit-code to a machine verdict (deterministic, pure). + + Contract (AC-1 / AC-3, hook exit-code contract 0/1/2): + * ``0`` -> ``"SUCCESS"`` (health-ok proven by the hook). + * ``1`` (rolled back), ``2`` (rollback also failed), anything else, or a + non-int/None -> ``"FAILED"`` (fail-closed; never advances on doubt). + """ + try: + code = int(exit_code) + except (TypeError, ValueError): + return "FAILED" + return "SUCCESS" if code == 0 else "FAILED" + + +def build_deploy_log(work_item_id: str, exit_code, status: str) -> str: + """Render a 14-deploy-log.md body whose ``deploy_status:`` frontmatter is the + verdict ``check_deploy_status`` / ``_parse_deploy_status`` reads (contract + unchanged, AC-10). The body is informational only — only the frontmatter is + machine-read. + """ + return ( + "---\n" + f"deploy_status: {status}\n" + f"work_item: {work_item_id}\n" + f"hook_exit_code: {exit_code}\n" + "deployed_by: deploy-finalizer\n" + "---\n\n" + "# Deploy log — ORCH-036 executable self-deploy\n\n" + f"Прод-деплой завершён хост-хуком с exit-code `{exit_code}` -> " + f"`deploy_status: {status}`.\n\n" + "Вердикт зафиксирован детерминированным finalizer'ом (Фаза C), не LLM.\n" + ) + + +# --------------------------------------------------------------------------- +# Sentinel state (restart-safe, no DB migration — ТЗ §4) +# --------------------------------------------------------------------------- +def _state_dir(base: str, repo: str, work_item_id: str | None) -> str: + return os.path.join(base, f".deploy-state-{repo}", (work_item_id or "_")) + + +def container_state_dir(repo: str, work_item_id: str | None) -> str: + """State dir as seen FROM THE CONTAINER (settings.repos_dir mount).""" + return _state_dir(settings.repos_dir, repo, work_item_id) + + +def host_state_dir(repo: str, work_item_id: str | None) -> str: + """State dir as seen FROM THE HOST (settings.host_repos_dir). + + Same physical directory as ``container_state_dir`` via the shared mount; the + host path is what we embed in the ssh command so the host wrapper writes the + ``result`` sentinel where the container can read it. + """ + return _state_dir(settings.host_repos_dir, repo, work_item_id) + + +def marker_path(repo: str, work_item_id: str | None, name: str) -> str: + return os.path.join(container_state_dir(repo, work_item_id), name) + + +def has_marker(repo: str, work_item_id: str | None, name: str) -> bool: + """True iff the named sentinel exists. Never raises.""" + try: + return os.path.isfile(marker_path(repo, work_item_id, name)) + except Exception as e: # noqa: BLE001 - never-raise + logger.warning("has_marker error for %s/%s/%s: %s", repo, work_item_id, name, e) + return False + + +def write_marker(repo: str, work_item_id: str | None, name: str, content: str = "") -> bool: + """Create/overwrite a sentinel (best-effort). Returns True on success.""" + try: + d = container_state_dir(repo, work_item_id) + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, name), "w", encoding="utf-8") as f: + f.write(str(content)) + return True + except OSError as e: + logger.warning("write_marker error for %s/%s/%s: %s", repo, work_item_id, name, e) + return False + + +def read_result(repo: str, work_item_id: str | None) -> tuple[bool, int | None]: + """Read the ``result`` sentinel (hook exit-code written by the host wrapper). + + Returns ``(present, exit_code)``: + * ``(False, None)`` -> not written yet (finalizer should DEFER); + * ``(True, )`` -> verdict ready; + * ``(True, 1)`` -> present but corrupt/unparseable -> treated as a + failure code (fail-closed) so we never advance on garbage. + Never raises. + """ + p = marker_path(repo, work_item_id, RESULT) + try: + with open(p, "r", encoding="utf-8") as f: + raw = f.read().strip() + except FileNotFoundError: + return False, None + except OSError as e: + logger.warning("read_result error for %s/%s: %s", repo, work_item_id, e) + return False, None + if raw == "": + return False, None + try: + return True, int(raw) + except ValueError: + logger.warning("read_result: corrupt result %r for %s/%s", raw, repo, work_item_id) + return True, 1 + + +# --------------------------------------------------------------------------- +# Detached host deploy: ssh + setsid (Phase B) +# --------------------------------------------------------------------------- +def build_deploy_command(repo: str, work_item_id: str | None, branch: str) -> list[str]: + """Build the ssh argv that launches the DETACHED prod deploy on the host. + + The remote command runs the hook via ``setsid`` with stdin/stdout detached and + backgrounded (``&``) so the process SURVIVES the prod container restart (BR-2), + then the WRAPPER (not the hook) writes the exit-code to the ``result`` sentinel: + + setsid bash -c 'cd && bash --deploy; \ + echo $? > ' >> 2>&1 `` makes the hook retag the + staging-validated image to the prod tag instead of rebuilding (no ``docker + build``). The exit-code contract of the hook is untouched. + """ + host_dir = host_state_dir(repo, work_item_id) + result_sentinel = os.path.join(host_dir, RESULT) + hook_log = os.path.join(host_dir, "hook.log") + + env_assignments = ( + f"SOURCE_IMAGE={shlex.quote(settings.deploy_prod_source_image)} " + f"TARGET_SERVICE={shlex.quote(settings.deploy_prod_target_service)} " + f"TARGET_PORT={int(settings.deploy_prod_target_port)} " + f"TARGET_IMAGE={shlex.quote(settings.deploy_prod_target_image)} " + f"COMPOSE_PROFILE={shlex.quote(settings.deploy_prod_compose_profile)} " + f"PREV_IMAGE_FILE={shlex.quote(settings.deploy_prod_prev_image_file)}" + ) + inner = ( + f"cd {shlex.quote(settings.deploy_host_repo_path)} && " + f"{env_assignments} " + f"bash {shlex.quote(settings.deploy_hook_script)} --deploy; " + f"echo $? > {shlex.quote(result_sentinel)}" + ) + remote = ( + f"setsid bash -c {shlex.quote(inner)} " + f">> {shlex.quote(hook_log)} 2>&1 tuple[bool, str]: + """Launch the detached prod deploy on the host (Phase B). Never raises. + + The ssh call returns immediately (the remote process is detached via setsid + + ``&``). Returns ``(True, msg)`` when ssh dispatched the detached process, or + ``(False, reason)`` so the caller can alert and let the human re-approve. + """ + # Ensure the shared state dir exists so the host wrapper can write `result`. + try: + os.makedirs(container_state_dir(repo, work_item_id), exist_ok=True) + except OSError as e: + logger.warning("initiate_deploy: state dir error for %s/%s: %s", repo, work_item_id, e) + + cmd = build_deploy_command(repo, work_item_id, branch) + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=_SSH_TIMEOUT) + except subprocess.TimeoutExpired: + return False, "ssh launch timeout" + except (subprocess.SubprocessError, OSError) as e: + return False, f"ssh launch error: {e}" + if r.returncode != 0: + detail = ((r.stderr or "") + (r.stdout or "")).strip()[:200] + return False, f"ssh launch failed (rc={r.returncode}): {detail}" + logger.info("initiate_deploy: detached prod deploy dispatched for %s/%s", repo, work_item_id) + return True, "deploy initiated (detached host process)" + + +# --------------------------------------------------------------------------- +# Deploy log write + best-effort merge (Phase C) +# --------------------------------------------------------------------------- +def write_deploy_log(repo: str, work_item_id: str, branch: str, exit_code, status: str) -> bool: + """Write 14-deploy-log.md into the task worktree (so check_deploy_status reads + it) and best-effort commit+push it. Returns True iff the file was written. + Never raises. + """ + from .git_worktree import get_worktree_path + + rel = f"docs/work-items/{work_item_id}/14-deploy-log.md" + try: + wt = get_worktree_path(repo, branch) + except Exception as e: # noqa: BLE001 - never-raise + logger.error("write_deploy_log: worktree error for %s/%s: %s", repo, branch, e) + return False + + path = os.path.join(wt, rel) + content = build_deploy_log(work_item_id, exit_code, status) + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(content) + except OSError as e: + logger.error("write_deploy_log: write error at %s: %s", path, e) + return False + + # Best-effort commit + push (the gate also falls back to origin/main). + git_env = { + **os.environ, + "HOME": "/home/slin", + "GIT_AUTHOR_NAME": "deploy-finalizer", + "GIT_AUTHOR_EMAIL": "deploy-finalizer@mva154.local", + "GIT_COMMITTER_NAME": "deploy-finalizer", + "GIT_COMMITTER_EMAIL": "deploy-finalizer@mva154.local", + } + try: + subprocess.run(["git", "-C", wt, "add", rel], + capture_output=True, timeout=_GIT_TIMEOUT, env=git_env) + commit = subprocess.run( + ["git", "-C", wt, "commit", "-m", + f"deploy(ORCH-036): finalize {status} for {work_item_id}"], + capture_output=True, text=True, timeout=_GIT_TIMEOUT, env=git_env, + ) + if commit.returncode == 0: + subprocess.run(["git", "-C", wt, "push", "origin", branch], + capture_output=True, timeout=_GIT_TIMEOUT, env=git_env) + except (subprocess.SubprocessError, OSError) as e: + logger.warning("write_deploy_log: git commit/push best-effort failed: %s", e) + return True diff --git a/src/stage_engine.py b/src/stage_engine.py index 63a1026..92ecdac 100644 --- a/src/stage_engine.py +++ b/src/stage_engine.py @@ -27,6 +27,7 @@ Agent-selection bug fix (ORCH-4): import logging import os +import time from dataclasses import dataclass, field from .db import get_db, update_task_stage, enqueue_job @@ -35,6 +36,7 @@ from .git_worktree import get_worktree_path from .review_parse import extract_review_findings, extract_test_failures from .qg.checks import QG_CHECKS from . import merge_gate +from . import self_deploy from .notifications import ( notify_stage_change, notify_qg_failure, @@ -190,6 +192,23 @@ def advance_stage( result.note = "terminal" return result + # --- ORCH-036 Phase B: human Approved on `deploy` -> initiate deploy -- + # A human flipping the Plane status to Approved on the `deploy` stage + # (finished_agent is None) is the prod-deploy trigger for the self-hosting + # repo. Initiate the DETACHED host deploy + enqueue the finalizer and + # return WITHOUT running check_deploy_status (the verdict does not exist + # yet — running the gate now would read a stale/absent log and falsely + # roll back, R-2). The finalizer (Phase C, finished_agent="deployer") + # records the verdict later; that path is NOT intercepted here. + if ( + current_stage == "deploy" + and finished_agent is None + and settings.deploy_require_manual_approve + and self_deploy.self_deploy_applies(repo) + ): + _handle_self_deploy_phase_b(task_id, repo, work_item_id, branch, result) + return result + # --- Quality gate ---------------------------------------------------- if qg_name and qg_name in QG_CHECKS: # Human-approval gate: split by path. @@ -252,6 +271,22 @@ def advance_stage( ): return result + # --- ORCH-036 Phase A: request approve before the prod deploy --------- + # On the deploy-staging -> deploy edge, AFTER a green check_staging_status + # and the merge-gate, the self-hosting repo does NOT auto-launch a prod + # deployer. Instead advance the STAGE to `deploy`, put the issue into an + # approval-pending state and wait for a human Approved (Phase B). The + # merge lease stays HELD across the wait (released on done / rollback). + if ( + current_stage == "deploy-staging" + and settings.deploy_require_manual_approve + and self_deploy.self_deploy_applies(repo) + ): + _handle_self_deploy_phase_a( + task_id, current_stage, repo, work_item_id, branch, result + ) + return result + # --- Advance --------------------------------------------------------- update_task_stage(task_id, next_stage) # Telegram live tracker: the analysis->architecture advance is the human @@ -762,3 +797,199 @@ def _handle_merge_gate_rollback( f"Task {task_id}: merge-gate FAILED, rolled back deploy-staging -> " f"development ({reason})" ) + + +# --------------------------------------------------------------------------- +# ORCH-036: executable self-deploy (Phase A/B/C) +# --------------------------------------------------------------------------- +def _handle_self_deploy_phase_a( + task_id, current_stage, repo, work_item_id, branch, result: AdvanceResult +): + """Phase A — advance to `deploy` and request a manual approve (no prod deploy). + + Staging is green and the branch is mergeable; for the self-hosting repo we do + NOT auto-deploy to prod. Move the task onto the `deploy` stage (so a later + human Approved lands there -> Phase B), set the issue approval-pending and ask + the human to flip the status to Approved. A restart-safe `approve-requested` + marker records that Phase A ran. The merge lease stays HELD. + """ + update_task_stage(task_id, "deploy") + notify_stage_change(task_id, current_stage, "deploy") + result.advanced = True + result.to_stage = "deploy" + result.note = "self-deploy-approval-pending" + + if work_item_id: + set_issue_in_review(work_item_id) + self_deploy.write_marker( + repo, work_item_id, self_deploy.APPROVE_REQUESTED, content=str(time.time()) + ) + if work_item_id: + plane_add_comment( + work_item_id, + "\U0001f7e1 Staging зелёный. Требуется ручной approve для ПРОД-деплоя: " + "смените статус задачи на «Approved», чтобы запустить деплой в прод (8500).", + author="deployer", + ) + send_telegram( + f"\U0001f7e1 {work_item_id}: staging OK. Ждёт approve на ПРОД-деплой " + f"(смените статус на Approved)." + ) + logger.info( + f"Task {task_id}: self-deploy Phase A — advanced to deploy, " + f"approval-pending (awaiting human Approved)" + ) + + +def _handle_self_deploy_phase_b(task_id, repo, work_item_id, branch, result: AdvanceResult): + """Phase B — a human Approved initiates the DETACHED prod deploy (idempotent). + + Idempotency-guard: if the `initiated` marker already exists (double Approved / + duplicate webhook, R-4) this is a no-op. Otherwise launch the detached host + deploy, and ONLY on success record `initiated` + enqueue the finalizer (so a + failed launch can be retried by re-approving). Returns without advancing — the + finalizer (Phase C) records the verdict once the hook finishes. + """ + if self_deploy.has_marker(repo, work_item_id, self_deploy.INITIATED): + result.note = "self-deploy-already-initiated" + logger.info( + f"Task {task_id}: prod deploy already initiated; ignoring repeat Approved" + ) + return + + ok, msg = self_deploy.initiate_deploy(repo, work_item_id, branch) + if not ok: + result.note = f"self-deploy-initiate-failed: {msg}" + if work_item_id: + plane_add_comment( + work_item_id, + f"⚠️ Не удалось запустить прод-деплой: {msg}. " + "Повторите approve после устранения причины.", + author="deployer", + ) + send_telegram(f"⚠️ {work_item_id}: прод-деплой не запустился: {msg}") + logger.error(f"Task {task_id}: self-deploy initiate failed: {msg}") + return + + self_deploy.write_marker( + repo, work_item_id, self_deploy.INITIATED, content=str(time.time()) + ) + task_desc = ( + f"Work item: {work_item_id}\nRepo: {repo}\nBranch: {branch}\n" + f"Stage: deploy\nNote: deploy-finalize poll (prod self-deploy initiated)." + ) + new_job = enqueue_job( + "deploy-finalizer", repo, task_desc, task_id=task_id, + available_at_delay_s=settings.deploy_finalize_delay_s, + ) + result.enqueued_agent = "deploy-finalizer" + result.enqueued_job_id = new_job + result.note = "self-deploy-initiated" + if work_item_id: + plane_add_comment( + work_item_id, + "\U0001f680 Прод-деплой стартовал (detached host-процесс). " + "Вердикт будет зафиксирован после health-check.", + author="deployer", + ) + send_telegram(f"\U0001f680 {work_item_id}: прод-деплой стартовал. Жду результат.") + logger.info( + f"Task {task_id}: self-deploy Phase B — detached deploy initiated, " + f"finalizer enqueued (job_id={new_job})" + ) + + +def _deploy_finalize_defer_count(task_id: int) -> int: + """How many times this task's finalizer has already deferred (restart-safe). + + Counted from the persisted jobs queue by the defer marker in task_content + (mirrors _merge_defer_count), so a service restart never resets the budget. + """ + conn = get_db() + n = conn.execute( + "SELECT COUNT(*) FROM jobs WHERE task_id=? AND task_content LIKE '%deploy-finalize defer%'", + (task_id,), + ).fetchone()[0] + conn.close() + return n + + +def run_deploy_finalizer(job: dict): + """Phase C — deterministic finalizer (reserved-agent `deploy-finalizer`, no LLM). + + Claimed by the worker in the NEW container after the prod restart. Reads the + `result` sentinel (hook exit-code written by the host wrapper): + * not written yet & budget left -> DEFER (re-queue with a delay); + * budget exhausted -> set_issue_blocked + Telegram (anti-livelock); + * present -> map exit-code -> deploy_status, write + 14-deploy-log.md, then advance_stage(finished_agent="deployer") so the + EXISTING contracts fire: SUCCESS -> terminal-sync deploy->done + release + lease; FAILED -> БАГ-8 rollback deploy->development + set_issue_blocked. + Never raises into the caller (the launcher marks the job done/failed). + """ + task_id = job.get("task_id") + repo = job.get("repo") + conn = get_db() + row = conn.execute( + "SELECT work_item_id, branch FROM tasks WHERE id=?", (task_id,) + ).fetchone() + conn.close() + if not row: + logger.error(f"deploy-finalizer: no task row for task_id={task_id}") + return + work_item_id, branch = row[0], row[1] + + present, code = self_deploy.read_result(repo, work_item_id) + if not present: + defers = _deploy_finalize_defer_count(task_id) + if defers < settings.deploy_finalize_max_attempts: + task_desc = ( + f"Work item: {work_item_id}\nRepo: {repo}\nBranch: {branch}\n" + f"Stage: deploy\nNote: deploy-finalize defer " + f"(attempt {defers + 1}/{settings.deploy_finalize_max_attempts}) — " + f"deploy result not ready, retrying after {settings.deploy_finalize_delay_s}s." + ) + new_job = enqueue_job( + "deploy-finalizer", repo, task_desc, task_id=task_id, + available_at_delay_s=settings.deploy_finalize_delay_s, + ) + logger.info( + f"Task {task_id}: deploy result not ready, finalizer deferred " + f"(job_id={new_job}, attempt {defers + 1}/{settings.deploy_finalize_max_attempts})" + ) + else: + if work_item_id: + set_issue_blocked(work_item_id) + send_telegram( + f"\U0001f6a8 {work_item_id}: deploy result не появился после " + f"{settings.deploy_finalize_max_attempts} попыток. Нужно ручное вмешательство." + ) + logger.error( + f"Task {task_id}: deploy-finalize defer attempts exhausted " + f"({settings.deploy_finalize_max_attempts})" + ) + return + + # Result present -> deterministic verdict. + status = self_deploy.map_exit_code_to_status(code) + self_deploy.write_deploy_log(repo, work_item_id, branch, code, status) + logger.info( + f"Task {task_id}: deploy finalized, hook exit={code} -> deploy_status={status}" + ) + if status == "SUCCESS" and work_item_id: + plane_add_comment( + work_item_id, + f"✅ Прод-деплой успешен (health-check OK, exit {code}).", + author="deployer", + ) + send_telegram(f"✅ {work_item_id}: прод-деплой успешен (exit {code}).") + + # Drive the EXISTING deploy contracts via the gate verdict we just wrote. + advance_stage( + task_id=task_id, + current_stage="deploy", + repo=repo, + work_item_id=work_item_id, + branch=branch, + finished_agent="deployer", + ) diff --git a/tests/test_deploy_approve.py b/tests/test_deploy_approve.py new file mode 100644 index 0000000..ee91ebd --- /dev/null +++ b/tests/test_deploy_approve.py @@ -0,0 +1,160 @@ +"""ORCH-036 TC-04/05/06: the manual-approve gate for the executable self-deploy. + +Contract (AC-5, AC-12): + * TC-04 — ``deploy_require_manual_approve`` defaults to True in settings. + * TC-05 — flag true + NO human approve -> the prod hook is NEVER called; the + deploy-staging -> deploy edge only advances the STAGE and requests an approve + (Phase A). ``initiate_deploy`` / ssh subprocess must not be touched. + * TC-06 — flag true + a human Approved -> the prod hook is launched EXACTLY once + (Phase B), idempotent on a repeated Approved (the ``initiated`` marker guards). +""" + +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_orch_deploy_approve.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import stage_engine # noqa: E402 +from src import self_deploy # noqa: E402 +from src.stage_engine import advance_stage # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + # Isolate the sentinel state dirs to a per-test tmp dir. + monkeypatch.setattr(self_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(self_deploy.settings, "host_repos_dir", str(tmp_path)) + yield + + +@pytest.fixture(autouse=True) +def silence_side_effects(monkeypatch): + for name in ( + "notify_stage_change", "notify_qg_failure", "notify_approve_requested", + "send_telegram", "plane_notify_stage", "plane_notify_qg", "plane_add_comment", + "set_issue_in_review", "set_issue_needs_input", "set_issue_in_progress", + "set_issue_blocked", "set_issue_done", + ): + monkeypatch.setattr(stage_engine, name, MagicMock()) + + +def _make_task(stage, repo="orchestrator", branch="feature/ORCH-036-x", wi="ORCH-036"): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + task_id = cur.lastrowid + conn.commit() + conn.close() + return task_id + + +def _stage(task_id): + conn = get_db() + row = conn.execute("SELECT stage FROM tasks WHERE id=?", (task_id,)).fetchone() + conn.close() + return row[0] + + +def _jobs(): + conn = get_db() + rows = conn.execute("SELECT agent, repo, task_id FROM jobs ORDER BY id").fetchall() + conn.close() + return [dict(r) for r in rows] + + +def _pass(*a, **k): + return (True, "ok") + + +# --------------------------------------------------------------------------- +# TC-04: default flag value +# --------------------------------------------------------------------------- +def test_tc04_manual_approve_default_true(): + """The fresh, un-overridden settings default must be True (safe-by-default).""" + from src.config import Settings + assert Settings().deploy_require_manual_approve is True + + +# --------------------------------------------------------------------------- +# TC-05: flag true, no approve -> prod hook NOT called (Phase A only) +# --------------------------------------------------------------------------- +def test_tc05_no_approve_does_not_call_prod_hook(monkeypatch): + monkeypatch.setattr(stage_engine.settings, "deploy_require_manual_approve", True) + monkeypatch.setattr( + stage_engine, "QG_CHECKS", + {**stage_engine.QG_CHECKS, + "check_staging_status": _pass, + "check_branch_mergeable": _pass}, + ) + # Spy: the deploy launcher must never run on the staging->deploy edge. + initiate = MagicMock() + monkeypatch.setattr(stage_engine.self_deploy, "initiate_deploy", initiate) + ssh_run = MagicMock() + monkeypatch.setattr(self_deploy.subprocess, "run", ssh_run) + + task_id = _make_task("deploy-staging") + res = advance_stage( + task_id, "deploy-staging", "orchestrator", "ORCH-036", + "feature/ORCH-036-x", finished_agent="deployer", + ) + + # Phase A: advanced the STAGE to deploy, but requested approve — no prod hook. + assert res.advanced is True + assert res.to_stage == "deploy" + assert _stage(task_id) == "deploy" + assert res.note == "self-deploy-approval-pending" + initiate.assert_not_called() + ssh_run.assert_not_called() + # No deployer job: the human Approved (Phase B) is what triggers the deploy. + assert _jobs() == [] + # The restart-safe approve-requested marker was written. + assert self_deploy.has_marker("orchestrator", "ORCH-036", self_deploy.APPROVE_REQUESTED) + + +# --------------------------------------------------------------------------- +# TC-06: flag true + Approved -> prod hook called exactly once (idempotent) +# --------------------------------------------------------------------------- +def test_tc06_approved_calls_prod_hook_exactly_once(monkeypatch): + monkeypatch.setattr(stage_engine.settings, "deploy_require_manual_approve", True) + monkeypatch.setattr(stage_engine.settings, "deploy_ssh_host", "mva154") + # Real initiate_deploy, but the ssh subprocess is mocked (rc=0 -> dispatched). + ssh_run = MagicMock(return_value=MagicMock(returncode=0, stdout="", stderr="")) + monkeypatch.setattr(self_deploy.subprocess, "run", ssh_run) + + task_id = _make_task("deploy") # already on deploy, awaiting Approved + + # 1st human Approved -> Phase B initiates the detached deploy. + res1 = advance_stage( + task_id, "deploy", "orchestrator", "ORCH-036", + "feature/ORCH-036-x", finished_agent=None, + ) + assert res1.note == "self-deploy-initiated" + assert ssh_run.call_count == 1 + # The finalizer was enqueued. + assert any(j["agent"] == "deploy-finalizer" for j in _jobs()) + assert self_deploy.has_marker("orchestrator", "ORCH-036", self_deploy.INITIATED) + + # 2nd (duplicate) Approved -> idempotent no-op, hook NOT called again. + res2 = advance_stage( + task_id, "deploy", "orchestrator", "ORCH-036", + "feature/ORCH-036-x", finished_agent=None, + ) + assert res2.note == "self-deploy-already-initiated" + assert ssh_run.call_count == 1 # still exactly one prod deploy diff --git a/tests/test_deploy_build_once.py b/tests/test_deploy_build_once.py new file mode 100644 index 0000000..1d797a0 --- /dev/null +++ b/tests/test_deploy_build_once.py @@ -0,0 +1,47 @@ +"""ORCH-036 TC-14: prod deploy is build-ONCE — retag the staging image, no rebuild (AC-7). + +The detached prod-deploy command must pass ``SOURCE_IMAGE=`` to the +hook so it retags the staging-validated image onto the prod tag instead of running +``docker build``. We assert the composed ssh command carries the staging source +image and never asks the hook to build. +""" + +import os + +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") + +from src import self_deploy # noqa: E402 + + +def test_tc14_deploy_command_retags_staging_image_no_build(monkeypatch): + monkeypatch.setattr(self_deploy.settings, "deploy_ssh_user", "slin") + monkeypatch.setattr(self_deploy.settings, "deploy_ssh_host", "mva154") + monkeypatch.setattr( + self_deploy.settings, "deploy_prod_source_image", "orchestrator-orchestrator-staging" + ) + + cmd = self_deploy.build_deploy_command("orchestrator", "ORCH-036", "feature/ORCH-036-x") + remote = cmd[-1] + + # The prevalidated staging image is handed to the hook as SOURCE_IMAGE (build-once). + assert "SOURCE_IMAGE=orchestrator-orchestrator-staging" in remote + # No rebuild is requested in the remote command. + assert "docker build" not in remote + assert "--build" not in remote + + +def test_tc14_hook_retag_branch_present(): + """The hook itself must honour SOURCE_IMAGE by retagging (no rebuild).""" + import pathlib + hook = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "orchestrator-deploy-hook.sh" + text = hook.read_text(encoding="utf-8") + assert 'SOURCE_IMAGE="${SOURCE_IMAGE:-}"' in text + # Build-once retag branch present; the hook never runs `docker build`. + assert 'docker tag "$SOURCE_IMAGE" "$TARGET_IMAGE"' in text + # No EXECUTABLE `docker build` line (comments mentioning it are fine). + exec_lines = [ + ln.strip() for ln in text.splitlines() + if ln.strip() and not ln.strip().startswith("#") + ] + assert not any("docker build" in ln for ln in exec_lines) diff --git a/tests/test_deploy_hook_mapping.py b/tests/test_deploy_hook_mapping.py new file mode 100644 index 0000000..96a843f --- /dev/null +++ b/tests/test_deploy_hook_mapping.py @@ -0,0 +1,47 @@ +"""ORCH-036 TC-01/02/03: deterministic exit-code -> deploy_status mapping. + +The finalizer (Phase C) maps the host-hook exit-code to the machine verdict via a +PURE function (no LLM, no I/O), so it is unit-testable in isolation. Contract +(hook exit-code 0/1/2, AC-1/AC-3): 0 -> SUCCESS; 1 (rolled back), 2 (rollback also +failed), and anything else -> FAILED (fail-closed). +""" + +import os + +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") + +from src.self_deploy import map_exit_code_to_status, build_deploy_log # noqa: E402 + + +def test_tc01_exit0_maps_to_success(): + assert map_exit_code_to_status(0) == "SUCCESS" + + +def test_tc02_exit1_rolled_back_maps_to_failed(): + assert map_exit_code_to_status(1) == "FAILED" + + +def test_tc03_exit2_rollback_also_failed_maps_to_failed(): + assert map_exit_code_to_status(2) == "FAILED" + + +def test_other_exit_codes_map_to_failed(): + for code in (3, 127, 255, -1): + assert map_exit_code_to_status(code) == "FAILED" + + +def test_non_int_or_none_maps_to_failed_fail_closed(): + assert map_exit_code_to_status(None) == "FAILED" + assert map_exit_code_to_status("garbage") == "FAILED" + + +def test_deploy_log_frontmatter_carries_status(): + """The rendered log must expose deploy_status in YAML frontmatter so the + existing _parse_deploy_status contract (AC-10) reads the right verdict.""" + body_ok = build_deploy_log("ORCH-036", 0, "SUCCESS") + assert body_ok.startswith("---\n") + assert "deploy_status: SUCCESS" in body_ok + body_fail = build_deploy_log("ORCH-036", 2, "FAILED") + assert "deploy_status: FAILED" in body_fail + assert "hook_exit_code: 2" in body_fail diff --git a/tests/test_deploy_hook_rollback_sim.py b/tests/test_deploy_hook_rollback_sim.py new file mode 100644 index 0000000..3fb25ec --- /dev/null +++ b/tests/test_deploy_hook_rollback_sim.py @@ -0,0 +1,118 @@ +"""ORCH-036 TC-19: deploy-hook auto-rollback simulation (AC-9). + +Drives the REAL ``scripts/orchestrator-deploy-hook.sh`` in a hermetic sandbox: +``docker`` / ``curl`` / ``git`` / ``sleep`` are replaced by PATH-shimmed stubs so +no real infra is touched (and prod is never restarted — INFRA safety). The curl +stub is stateful: the freshly-deployed service is UNHEALTHY for the whole deploy +health-check window, which must trigger the hook's AUTO-ROLLBACK; after the +rollback restart the previous image is HEALTHY again. + +Expected hook contract (exit-code 0/1/2): + * health fails -> auto rollback -> previous image healthy -> exit 1 (rolled back); + * the whole run completes well under the 60s MTTR budget (sleeps are shimmed). +""" + +import os +import shutil +import stat +import subprocess +import time + +import pytest + +HOOK = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "scripts", "orchestrator-deploy-hook.sh", +) + +pytestmark = pytest.mark.skipif( + shutil.which("bash") is None, reason="bash required for hook simulation" +) + + +def _write_exec(path, content): + with open(path, "w", encoding="utf-8") as f: + f.write(content) + os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +def _setup_sandbox(tmp_path): + """Create PATH-shimmed docker/curl/git/sleep stubs + a rewritten hook copy.""" + binx = tmp_path / "bin" + binx.mkdir() + state = tmp_path / "state" + state.mkdir() + repo = tmp_path / "repo" + repo.mkdir() + cnt = state / "curl_count" + + # docker: fake a running service + a recoverable previous image. + _write_exec(str(binx / "docker"), """#!/bin/bash +case "$1" in + compose) + for a in "$@"; do [ "$a" = "ps" ] && { echo "fakecid"; exit 0; }; done + exit 0;; + inspect) echo "sha256:previmage"; exit 0;; + image) exit 0;; # docker image inspect -> found + tag) exit 0;; + *) exit 0;; +esac +""") + + # curl: first 20 invocations (10 deploy health attempts x2 calls) UNHEALTHY, + # then HEALTHY (the rolled-back previous image). + _write_exec(str(binx / "curl"), f"""#!/bin/bash +CNT="{cnt}" +n=$(cat "$CNT" 2>/dev/null || echo 0); n=$((n+1)); echo "$n" > "$CNT" +iscode="" +for a in "$@"; do [ "$a" = "-w" ] && iscode=1; done +if [ "$n" -gt 20 ]; then + [ -n "$iscode" ] && echo "200" || echo '{{"status":"ok"}}' +else + [ -n "$iscode" ] && echo "000" || echo "" +fi +exit 0 +""") + + _write_exec(str(binx / "git"), "#!/bin/bash\nexit 0\n") + # Shim sleep to a no-op so the simulation runs fast (real timing is governed + # by the hook's sleep args; here we only assert the rollback CONTROL FLOW). + _write_exec(str(binx / "sleep"), "#!/bin/bash\nexit 0\n") + + # Copy the hook, repointing REPO to the sandbox (avoids the hardcoded prod path). + hook_text = open(HOOK, encoding="utf-8").read() + hook_text = hook_text.replace( + "REPO=/home/slin/repos/orchestrator", f"REPO={repo}" + ) + hook_copy = tmp_path / "hook.sh" + _write_exec(str(hook_copy), hook_text) + + env = { + **os.environ, + "PATH": f"{binx}:{os.environ['PATH']}", + "LOG": str(state / "hook.log"), + "PREV_IMAGE_FILE": str(state / "prev-image"), + "COMPOSE_PROFILE": "staging", + "TARGET_SERVICE": "orchestrator-staging", + "TARGET_PORT": "8501", + } + return hook_copy, env + + +def test_tc19_unhealthy_deploy_auto_rolls_back_exit1(tmp_path): + hook_copy, env = _setup_sandbox(tmp_path) + + t0 = time.time() + proc = subprocess.run( + ["bash", str(hook_copy), "--deploy"], + env=env, capture_output=True, text=True, timeout=60, + ) + elapsed = time.time() - t0 + + # AC-9: unhealthy deploy -> auto rollback succeeded on the previous image -> exit 1. + assert proc.returncode == 1, f"stdout={proc.stdout}\nstderr={proc.stderr}" + out = proc.stdout + proc.stderr + assert "AUTO ROLLBACK" in out + assert "rolled back to previous image successfully" in out + # MTTR well under the 60s budget (sleeps shimmed; control flow only). + assert elapsed < 60 diff --git a/tests/test_deploy_notifications.py b/tests/test_deploy_notifications.py new file mode 100644 index 0000000..b423a41 --- /dev/null +++ b/tests/test_deploy_notifications.py @@ -0,0 +1,102 @@ +"""ORCH-036 TC-12/13: no silent deploy — both Plane AND Telegram are notified (AC-6). + +The finalizer (Phase C) must announce the prod-deploy outcome on BOTH channels: + * TC-12 — a SUCCESS deploy -> a Plane comment AND a Telegram message. + * TC-13 — a FAILED deploy (rollback) -> a Plane comment AND a Telegram message. +""" + +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_orch_deploy_notif.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import stage_engine # noqa: E402 +from src import self_deploy # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr(self_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(self_deploy.settings, "host_repos_dir", str(tmp_path)) + monkeypatch.setattr(stage_engine.self_deploy, "write_deploy_log", MagicMock(return_value=True)) + monkeypatch.setattr(stage_engine.merge_gate, "release_merge_lease", MagicMock()) + yield + + +@pytest.fixture(autouse=True) +def silence_side_effects(monkeypatch): + for name in ( + "notify_stage_change", "notify_qg_failure", "notify_approve_requested", + "send_telegram", "plane_notify_stage", "plane_notify_qg", "plane_add_comment", + "set_issue_in_review", "set_issue_needs_input", "set_issue_in_progress", + "set_issue_blocked", "set_issue_done", + ): + monkeypatch.setattr(stage_engine, name, MagicMock()) + + +def _make_task(stage, repo="orchestrator", branch="feature/ORCH-036-x", wi="ORCH-036"): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + task_id = cur.lastrowid + conn.commit() + conn.close() + return task_id + + +def _pass(*a, **k): + return (True, "ok") + + +def _fail(reason): + def _f(*a, **k): + return (False, reason) + return _f + + +def _run_finalizer(task_id): + stage_engine.run_deploy_finalizer( + {"task_id": task_id, "repo": "orchestrator", "id": 1, "agent": "deploy-finalizer"} + ) + + +def test_tc12_success_notifies_plane_and_telegram(monkeypatch): + self_deploy.write_marker("orchestrator", "ORCH-036", self_deploy.RESULT, "0") + monkeypatch.setattr( + stage_engine, "QG_CHECKS", + {**stage_engine.QG_CHECKS, "check_deploy_status": _pass}, + ) + task_id = _make_task("deploy") + _run_finalizer(task_id) + assert stage_engine.plane_add_comment.called + assert stage_engine.send_telegram.called + + +def test_tc13_rollback_notifies_plane_and_telegram(monkeypatch): + self_deploy.write_marker("orchestrator", "ORCH-036", self_deploy.RESULT, "1") + monkeypatch.setattr( + stage_engine, "QG_CHECKS", + {**stage_engine.QG_CHECKS, "check_deploy_status": _fail("Deploy status: FAILED")}, + ) + task_id = _make_task("deploy") + _run_finalizer(task_id) + # The БАГ-8 rollback path announces on both channels (no silent failure). + assert stage_engine.send_telegram.called + assert stage_engine.plane_add_comment.called or stage_engine.plane_notify_qg.called diff --git a/tests/test_deploy_rollback.py b/tests/test_deploy_rollback.py new file mode 100644 index 0000000..845ab2e --- /dev/null +++ b/tests/test_deploy_rollback.py @@ -0,0 +1,100 @@ +"""ORCH-036 TC-10: a FAILED prod deploy rolls back deploy -> development (AC-4). + +The finalizer (Phase C) reads the hook ``result`` sentinel, maps a non-zero exit +to ``deploy_status: FAILED`` and then drives the EXISTING deploy contract via +``advance_stage(finished_agent="deployer")``. With a FAILED verdict the БАГ-8 +rollback fires: deploy -> development, ``set_issue_blocked`` + Telegram alert, and +(for the self-hosting repo) the merge-lease is released so the branch is not +wedged. The hook exit-code -> verdict mapping is unit-tested in +``test_deploy_hook_mapping.py``; here we assert the engine REACTION. +""" + +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_orch_deploy_rollback.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import stage_engine # noqa: E402 +from src import self_deploy # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr(self_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(self_deploy.settings, "host_repos_dir", str(tmp_path)) + # The finalizer's deploy-log write touches a git worktree we don't have here; + # the verdict it drives comes from check_deploy_status (monkeypatched below). + monkeypatch.setattr(stage_engine.self_deploy, "write_deploy_log", MagicMock(return_value=True)) + yield + + +@pytest.fixture(autouse=True) +def silence_side_effects(monkeypatch): + for name in ( + "notify_stage_change", "notify_qg_failure", "notify_approve_requested", + "send_telegram", "plane_notify_stage", "plane_notify_qg", "plane_add_comment", + "set_issue_in_review", "set_issue_needs_input", "set_issue_in_progress", + "set_issue_blocked", "set_issue_done", + ): + monkeypatch.setattr(stage_engine, name, MagicMock()) + + +def _make_task(stage, repo="orchestrator", branch="feature/ORCH-036-x", wi="ORCH-036"): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + task_id = cur.lastrowid + conn.commit() + conn.close() + return task_id + + +def _stage(task_id): + conn = get_db() + row = conn.execute("SELECT stage FROM tasks WHERE id=?", (task_id,)).fetchone() + conn.close() + return row[0] + + +def _fail(reason): + def _f(*a, **k): + return (False, reason) + return _f + + +def test_tc10_failed_deploy_rolls_back_to_development(monkeypatch): + # Hook reported exit 1 (rolled back) -> the host wrapper wrote result=1. + self_deploy.write_marker("orchestrator", "ORCH-036", self_deploy.RESULT, "1") + # The deploy-log verdict the gate reads is FAILED. + monkeypatch.setattr( + stage_engine, "QG_CHECKS", + {**stage_engine.QG_CHECKS, "check_deploy_status": _fail("Deploy status: FAILED")}, + ) + task_id = _make_task("deploy") + + stage_engine.run_deploy_finalizer( + {"task_id": task_id, "repo": "orchestrator", "id": 1, "agent": "deploy-finalizer"} + ) + + # БАГ-8 rollback fired: NOT done, back on development, blocked + alerted. + assert _stage(task_id) == "development" + assert stage_engine.set_issue_blocked.called + assert stage_engine.send_telegram.called + assert stage_engine.set_issue_done.called is False diff --git a/tests/test_deploy_routing.py b/tests/test_deploy_routing.py new file mode 100644 index 0000000..9ae80dc --- /dev/null +++ b/tests/test_deploy_routing.py @@ -0,0 +1,174 @@ +"""ORCH-036 TC-07/08/09: self vs non-self deploy routing (AC-2, AC-11). + + * TC-07 — ``is_self_hosting_repo``/``self_deploy_applies`` recognise the + orchestrator repo and reject any other (no regression). + * TC-08 — for the self repo the restart is launched as a DETACHED host process + (ssh + setsid + background), never synchronously inside the agent. + * TC-09 — for a non-self repo (enduro-trails) the deploy keeps the legacy path: + the self-deploy Phase A/B logic does NOT apply. +""" + +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_orch_deploy_routing.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import stage_engine # noqa: E402 +from src import self_deploy # noqa: E402 +from src.qg.checks import is_self_hosting_repo # noqa: E402 +from src.stage_engine import advance_stage # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr(self_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(self_deploy.settings, "host_repos_dir", str(tmp_path)) + yield + + +@pytest.fixture(autouse=True) +def silence_side_effects(monkeypatch): + for name in ( + "notify_stage_change", "notify_qg_failure", "notify_approve_requested", + "send_telegram", "plane_notify_stage", "plane_notify_qg", "plane_add_comment", + "set_issue_in_review", "set_issue_needs_input", "set_issue_in_progress", + "set_issue_blocked", "set_issue_done", + ): + monkeypatch.setattr(stage_engine, name, MagicMock()) + + +def _make_task(stage, repo, branch, wi): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + task_id = cur.lastrowid + conn.commit() + conn.close() + return task_id + + +def _stage(task_id): + conn = get_db() + row = conn.execute("SELECT stage FROM tasks WHERE id=?", (task_id,)).fetchone() + conn.close() + return row[0] + + +def _jobs(): + conn = get_db() + rows = conn.execute("SELECT agent, repo, task_id FROM jobs ORDER BY id").fetchall() + conn.close() + return [dict(r) for r in rows] + + +def _pass(*a, **k): + return (True, "ok") + + +# --------------------------------------------------------------------------- +# TC-07: routing predicate +# --------------------------------------------------------------------------- +def test_tc07_is_self_hosting_repo_only_orchestrator(): + assert is_self_hosting_repo("orchestrator") is True + assert is_self_hosting_repo("ORCHESTRATOR") is True # case-insensitive + assert is_self_hosting_repo("enduro-trails") is False + assert is_self_hosting_repo("") is False + assert is_self_hosting_repo(None) is False + + +def test_tc07_self_deploy_applies_mirrors_routing(monkeypatch): + monkeypatch.setattr(self_deploy.settings, "self_deploy_enabled", True) + monkeypatch.setattr(self_deploy.settings, "self_deploy_repos", "") + assert self_deploy.self_deploy_applies("orchestrator") is True + assert self_deploy.self_deploy_applies("enduro-trails") is False + # Global kill-switch wins. + monkeypatch.setattr(self_deploy.settings, "self_deploy_enabled", False) + assert self_deploy.self_deploy_applies("orchestrator") is False + + +# --------------------------------------------------------------------------- +# TC-08: self repo -> DETACHED host process (ssh + setsid + background) +# --------------------------------------------------------------------------- +def test_tc08_self_repo_launches_detached_host_process(monkeypatch): + """The deploy command must be an ssh invocation that detaches the hook via + setsid and backgrounds it (`&`), so it survives the prod container restart — + i.e. NOT a synchronous in-agent call.""" + monkeypatch.setattr(self_deploy.settings, "deploy_ssh_user", "slin") + monkeypatch.setattr(self_deploy.settings, "deploy_ssh_host", "mva154") + + cmd = self_deploy.build_deploy_command("orchestrator", "ORCH-036", "feature/ORCH-036-x") + + assert cmd[0] == "ssh" + assert "slin@mva154" in cmd + remote = cmd[-1] + assert "setsid" in remote # detached session + assert remote.rstrip().endswith("&") # backgrounded + assert " legacy path, self-deploy logic does not apply +# --------------------------------------------------------------------------- +def test_tc09_non_self_repo_uses_legacy_path(monkeypatch): + """enduro-trails on the deploy-staging -> deploy edge: no Phase A interception, + the deployer is enqueued for the deploy stage exactly as before ORCH-036.""" + monkeypatch.setattr(stage_engine.settings, "deploy_require_manual_approve", True) + monkeypatch.setattr( + stage_engine, "QG_CHECKS", + {**stage_engine.QG_CHECKS, "check_staging_status": _pass}, + ) # check_branch_mergeable left REAL -> N/A for non-self repo + # Spy: self-deploy must not be initiated for a non-self repo. + initiate = MagicMock() + monkeypatch.setattr(stage_engine.self_deploy, "initiate_deploy", initiate) + + task_id = _make_task("deploy-staging", "enduro-trails", "feature/ET-009-x", "ET-009") + res = advance_stage( + task_id, "deploy-staging", "enduro-trails", "ET-009", + "feature/ET-009-x", finished_agent="deployer", + ) + + assert res.advanced is True + assert _stage(task_id) == "deploy" + assert res.note != "self-deploy-approval-pending" + initiate.assert_not_called() + # Legacy path enqueues the deployer for the deploy stage. + jobs = _jobs() + assert len(jobs) == 1 + assert jobs[0]["agent"] == "deployer" + # No self-deploy marker for the non-self repo. + assert not self_deploy.has_marker("enduro-trails", "ET-009", self_deploy.APPROVE_REQUESTED) diff --git a/tests/test_deploy_terminal_sync.py b/tests/test_deploy_terminal_sync.py new file mode 100644 index 0000000..5aae57e --- /dev/null +++ b/tests/test_deploy_terminal_sync.py @@ -0,0 +1,104 @@ +"""ORCH-036 TC-17: a SUCCESS prod deploy preserves the terminal-sync contract (AC-10). + +When the finalizer (Phase C) reads exit 0 -> ``deploy_status: SUCCESS`` and drives +``advance_stage(finished_agent="deployer")``, the EXISTING deploy->done transition +must still fire unchanged: stage becomes ``done``, ``set_issue_done`` is called, no +agent is launched, and the merge-lease is released (terminal-sync, ORCH-43/БАГ-8 +contract). ORCH-036 only changes HOW the verdict is produced, never the contract. +""" + +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_orch_deploy_terminal.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import stage_engine # noqa: E402 +from src import self_deploy # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr(self_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(self_deploy.settings, "host_repos_dir", str(tmp_path)) + monkeypatch.setattr(stage_engine.self_deploy, "write_deploy_log", MagicMock(return_value=True)) + yield + + +@pytest.fixture(autouse=True) +def silence_side_effects(monkeypatch): + for name in ( + "notify_stage_change", "notify_qg_failure", "notify_approve_requested", + "send_telegram", "plane_notify_stage", "plane_notify_qg", "plane_add_comment", + "set_issue_in_review", "set_issue_needs_input", "set_issue_in_progress", + "set_issue_blocked", "set_issue_done", + ): + monkeypatch.setattr(stage_engine, name, MagicMock()) + + +def _make_task(stage, repo="orchestrator", branch="feature/ORCH-036-x", wi="ORCH-036"): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + task_id = cur.lastrowid + conn.commit() + conn.close() + return task_id + + +def _stage(task_id): + conn = get_db() + row = conn.execute("SELECT stage FROM tasks WHERE id=?", (task_id,)).fetchone() + conn.close() + return row[0] + + +def _jobs(): + conn = get_db() + rows = conn.execute("SELECT agent FROM jobs ORDER BY id").fetchall() + conn.close() + return [r[0] for r in rows] + + +def _pass(*a, **k): + return (True, "ok") + + +def test_tc17_success_deploy_syncs_terminal_done(monkeypatch): + # Hook reported exit 0 -> the host wrapper wrote result=0. + self_deploy.write_marker("orchestrator", "ORCH-036", self_deploy.RESULT, "0") + monkeypatch.setattr( + stage_engine, "QG_CHECKS", + {**stage_engine.QG_CHECKS, "check_deploy_status": _pass}, + ) + # Spy the merge-lease release to confirm the terminal-sync still frees it. + release = MagicMock() + monkeypatch.setattr(stage_engine.merge_gate, "release_merge_lease", release) + + task_id = _make_task("deploy") + stage_engine.run_deploy_finalizer( + {"task_id": task_id, "repo": "orchestrator", "id": 1, "agent": "deploy-finalizer"} + ) + + assert _stage(task_id) == "done" + assert stage_engine.set_issue_done.called + # The merge-lease is released on the deploy->done terminal-sync. + release.assert_called_once_with("orchestrator", "feature/ORCH-036-x") + # No agent is launched leaving deploy (terminal). + assert _jobs() == [] diff --git a/tests/test_qg_checks.py b/tests/test_qg_checks.py new file mode 100644 index 0000000..c6bfb01 --- /dev/null +++ b/tests/test_qg_checks.py @@ -0,0 +1,53 @@ +"""ORCH-036 TC-15: the deploy-verdict parse contract is unchanged (AC-10). + +``_parse_deploy_status`` reads ONLY the machine-readable ``deploy_status:`` YAML +frontmatter (never prose). ORCH-036 produces the verdict differently (a +deterministic finalizer instead of an LLM), but the parse contract that the gate +relies on must remain bit-identical: + SUCCESS -> (True, ...), FAILED -> (False, ...), no/!frontmatter -> (False, ...). +""" + +import os + +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") + +from src.qg.checks import _parse_deploy_status # noqa: E402 +from src.self_deploy import build_deploy_log # noqa: E402 + + +def test_tc15_success_frontmatter_passes(): + ok, reason = _parse_deploy_status("---\ndeploy_status: SUCCESS\n---\n\nbody") + assert ok is True + assert "SUCCESS" in reason + + +def test_tc15_failed_frontmatter_fails(): + ok, reason = _parse_deploy_status("---\ndeploy_status: FAILED\n---\n\nbody") + assert ok is False + assert "FAILED" in reason + + +def test_tc15_no_frontmatter_fails(): + ok, _ = _parse_deploy_status("just prose, deploy_status: SUCCESS in text but no frontmatter") + assert ok is False + + +def test_tc15_missing_field_fails(): + ok, _ = _parse_deploy_status("---\nother_field: SUCCESS\n---\n") + assert ok is False + + +def test_tc15_prose_success_word_does_not_pass(): + """Defensive: the word SUCCESS in prose must NOT satisfy the gate.""" + ok, _ = _parse_deploy_status("# Deploy\n\nDeploy was a SUCCESS, hooray!\n") + assert ok is False + + +def test_tc15_finalizer_log_roundtrips_through_parser(): + """The finalizer's rendered log must be readable by the EXISTING parser — + SUCCESS passes, FAILED fails — proving the producer/consumer contract holds.""" + ok_s, _ = _parse_deploy_status(build_deploy_log("ORCH-036", 0, "SUCCESS")) + ok_f, _ = _parse_deploy_status(build_deploy_log("ORCH-036", 2, "FAILED")) + assert ok_s is True + assert ok_f is False diff --git a/tests/test_stage_engine.py b/tests/test_stage_engine.py index 678dcad..89229b5 100644 --- a/tests/test_stage_engine.py +++ b/tests/test_stage_engine.py @@ -822,7 +822,12 @@ class TestMergeGate: def test_tc20_pass_advances_to_deploy(self, monkeypatch): """TC-20 / AC-1: gate PASS (rebased + green) -> advance to deploy, deployer - enqueued, NO rollback. staging gate must pass first (same edge).""" + enqueued, NO rollback. staging gate must pass first (same edge). + + ORCH-036: disable the manual-approve self-deploy interception so this test + keeps exercising the merge-gate in isolation (the executable self-deploy + Phase A path is covered separately in test_deploy_approve.py).""" + monkeypatch.setattr(stage_engine.settings, "deploy_require_manual_approve", False) monkeypatch.setattr( stage_engine, "QG_CHECKS", {**stage_engine.QG_CHECKS, diff --git a/tests/test_stages.py b/tests/test_stages.py new file mode 100644 index 0000000..1ecaf7a --- /dev/null +++ b/tests/test_stages.py @@ -0,0 +1,41 @@ +"""ORCH-036 TC-16: STAGE_TRANSITIONS for deploy are unchanged (AC-10). + +ORCH-036 only changes HOW the deploy verdict is produced (a deterministic +finalizer) — it must NOT touch the state machine. The deploy edge keeps its +exact transition (deploy -> done), no in-line agent (None), and the gate +``check_deploy_status``. The deploy-staging edge is likewise untouched. +""" + +import os + +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") + +from src.stages import ( # noqa: E402 + STAGE_TRANSITIONS, + get_agent_for_stage, + get_next_stage, + get_qg_for_stage, +) + + +def test_tc16_deploy_transition_unchanged(): + assert STAGE_TRANSITIONS["deploy"] == { + "next": "done", "agent": None, "qg": "check_deploy_status" + } + assert get_next_stage("deploy") == "done" + assert get_agent_for_stage("deploy") is None + assert get_qg_for_stage("deploy") == "check_deploy_status" + + +def test_tc16_deploy_staging_transition_unchanged(): + assert STAGE_TRANSITIONS["deploy-staging"] == { + "next": "deploy", "agent": "deployer", "qg": "check_staging_status" + } + assert get_next_stage("deploy-staging") == "deploy" + assert get_agent_for_stage("deploy-staging") == "deployer" + assert get_qg_for_stage("deploy-staging") == "check_staging_status" + + +def test_tc16_done_is_terminal(): + assert get_next_stage("done") is None diff --git a/tests/test_staging_precondition.py b/tests/test_staging_precondition.py new file mode 100644 index 0000000..641a933 --- /dev/null +++ b/tests/test_staging_precondition.py @@ -0,0 +1,99 @@ +"""ORCH-036 TC-11: the staging precondition is preserved (AC-8). + +A red staging gate (``staging_status: FAILED``) must roll the task back to +development and NEVER let it reach the ``deploy`` stage — so the executable +prod self-deploy can never be initiated off a failed staging run. ORCH-036 adds +its Phase A interception AFTER ``check_staging_status``, so a staging failure +short-circuits before any self-deploy logic runs. +""" + +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_orch_staging_precond.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import stage_engine # noqa: E402 +from src import self_deploy # noqa: E402 +from src.stage_engine import advance_stage # noqa: E402 + + +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr(self_deploy.settings, "repos_dir", str(tmp_path)) + monkeypatch.setattr(self_deploy.settings, "host_repos_dir", str(tmp_path)) + yield + + +@pytest.fixture(autouse=True) +def silence_side_effects(monkeypatch): + for name in ( + "notify_stage_change", "notify_qg_failure", "notify_approve_requested", + "send_telegram", "plane_notify_stage", "plane_notify_qg", "plane_add_comment", + "set_issue_in_review", "set_issue_needs_input", "set_issue_in_progress", + "set_issue_blocked", "set_issue_done", + ): + monkeypatch.setattr(stage_engine, name, MagicMock()) + + +def _make_task(stage, repo="orchestrator", branch="feature/ORCH-036-x", wi="ORCH-036"): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) " + "VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + task_id = cur.lastrowid + conn.commit() + conn.close() + return task_id + + +def _stage(task_id): + conn = get_db() + row = conn.execute("SELECT stage FROM tasks WHERE id=?", (task_id,)).fetchone() + conn.close() + return row[0] + + +def _fail(reason): + def _f(*a, **k): + return (False, reason) + return _f + + +def test_tc11_staging_failed_never_reaches_deploy(monkeypatch): + monkeypatch.setattr(stage_engine.settings, "deploy_require_manual_approve", True) + monkeypatch.setattr( + stage_engine, "QG_CHECKS", + {**stage_engine.QG_CHECKS, + "check_staging_status": _fail("Staging status: FAILED")}, + ) + # Guard: a failed staging run must not trigger any self-deploy logic. + initiate = MagicMock() + monkeypatch.setattr(stage_engine.self_deploy, "initiate_deploy", initiate) + + task_id = _make_task("deploy-staging") + res = advance_stage( + task_id, "deploy-staging", "orchestrator", "ORCH-036", + "feature/ORCH-036-x", finished_agent="deployer", + ) + + assert res.advanced is False + assert res.rolled_back_to == "development" + assert _stage(task_id) == "development" # NEVER reached deploy + initiate.assert_not_called() + assert not self_deploy.has_marker("orchestrator", "ORCH-036", self_deploy.APPROVE_REQUESTED)