From cc41dd849c37b8a8aae646e83e01ce438ca936bb Mon Sep 17 00:00:00 2001 From: claude-bot Date: Tue, 16 Jun 2026 08:42:36 +0300 Subject: [PATCH] fix(staging): host-side ssh execution + env classification for staging-runner (ORCH-123) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ORCH-115 deterministic staging-runner ran `docker exec` FROM INSIDE the prod `orchestrator` container, which ships only `openssh-client git curl` — no `docker` CLI (Dockerfile:11). `Popen(["docker", ...])` hit FileNotFoundError -> a PERMANENT environment defect that was mis-routed as a code-fail rollback `deploy-staging -> development` (burning developer-retries). Incident ORCH-116: every self-hosting task reaching deploy-staging was doomed to a false rollback. Fix (adr-0049, additive, flag-gated, never-raise, self-hosting scope; the gate / artifact contract / STAGE_TRANSITIONS / DB schema are byte-for-byte unchanged): - D1: build_staging_command() wraps the SAME `docker exec ... staging_check.py ... --mode stub` in `ssh '<...>'` so it runs HOST-SIDE over the existing trusted ssh channel (mirror self_deploy / image_freshness). New flag staging_runner_exec_host_side (default True). No docker CLI/SDK added to the image, docker.sock not used in-container (D2 security). - D3: three-way classify_staging_outcome (suite-ran / permanent-env / transient-infra), disambiguating the exit=1 collision by scanning stderr. - D4: invariant "infra != code-fail" — permanent-env / exhausted transient-infra end in an infra-HOLD (no rollback, no developer-retry), NOT a false FAILED rollback (supersedes ORCH-115 D5). A really-executed failing suite still rolls back (anti-over-tolerance). R-2 verified: a held deploy-staging task is not rolled back by the reconciler. - D5: prod-like preflight() of the host-side channel at startup (main.lifespan, best-effort, never blocks). - D8: snapshot adds permanent_env / exec_host_side / preflight. Docs (golden source, same PR): INFRA.md execution-boundary section, architecture/README.md, CLAUDE.md, CHANGELOG.md, .env.example. Tests: tests/test_orch123_staging_runner_exec.py (TC-01 mandatory regression red->green; TC-02..TC-14 + R-2). ORCH-115 anti-drift green (3 tests updated for the D1/D4/D8 supersession). Full suite: 2131 passed. Refs: ORCH-123 Co-Authored-By: Claude Opus 4.8 --- .env.example | 11 +- CHANGELOG.md | 7 + CLAUDE.md | 36 ++ docs/architecture/README.md | 2 +- docs/operations/INFRA.md | 29 ++ src/main.py | 15 + src/staging_runner.py | 341 +++++++++++++-- tests/test_orch115_staging_runner.py | 25 +- tests/test_orch123_staging_runner_exec.py | 503 ++++++++++++++++++++++ 9 files changed, 917 insertions(+), 52 deletions(-) create mode 100644 tests/test_orch123_staging_runner_exec.py diff --git a/.env.example b/.env.example index b27f915..a4d4bb1 100644 --- a/.env.example +++ b/.env.example @@ -569,14 +569,21 @@ ORCH_COVERAGE_RUN_TIMEOUT_S=900 # STAGING_RUNNER_REPOS -> CSV scope; empty -> self-hosting only. # STAGING_RUNNER_TIMEOUT_S -> wall-clock budget for the docker-exec suite # (malformed/non-positive -> default 600 + WARNING). -# STAGING_RUNNER_INFRA_MAX_RETRIES -> tool-error (suite did NOT execute) bounded DEFER -# budget before a fail-closed FAILED (anti ORCH-110). +# STAGING_RUNNER_INFRA_MAX_RETRIES -> transient-infra (timeout/ssh) bounded DEFER +# budget before an infra-HOLD (anti ORCH-110). # STAGING_RUNNER_INFRA_RETRY_DELAY_S-> delay before the re-queued deployer job. +# STAGING_RUNNER_EXEC_HOST_SIDE -> ORCH-123 (adr-0049): true (default = prod) wraps +# the `docker exec` in `ssh '<...>'` so +# the suite runs HOST-SIDE (the prod container ships +# no docker CLI; incident ORCH-116). false -> the +# prior in-container `docker exec` (valid only where +# a docker CLI is baked into the image). Rollback knob. ORCH_STAGING_RUNNER_ENABLED=true ORCH_STAGING_RUNNER_REPOS= ORCH_STAGING_RUNNER_TIMEOUT_S=600 ORCH_STAGING_RUNNER_INFRA_MAX_RETRIES=2 ORCH_STAGING_RUNNER_INFRA_RETRY_DELAY_S=30 +ORCH_STAGING_RUNNER_EXEC_HOST_SIDE=true # ORCH-057 (follow-up ORCH-040): legacy root-owned ownership detect + actionable # worktree error. After the uid migration (user: "1000:1000") legacy root:root files diff --git a/CHANGELOG.md b/CHANGELOG.md index 2902751..b64b721 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ Формат: [Keep a Changelog](https://keepachangelog.com/). Записи — на смысловой PR/задачу. ## [Unreleased] +- **Host-side исполнение staging-раннера + классификация environment-дефекта** (ORCH-123, `fix`, bug→escalate full-cycle): устранён инцидент **ORCH-116** — детерминированный staging-раннер (ORCH-115) вызывал `docker exec` **изнутри** прод-контейнера `orchestrator`, где **нет бинаря `docker`** (образ несёт только `openssh-client git curl`, `Dockerfile:11`; `/var/run/docker.sock` смонтирован, но клиента нет) → `Popen(["docker", …])` падал `FileNotFoundError` → ветка tool-error → инфра-DEFER×2 → fail-closed `FAILED` → **ложный** откат `deploy-staging → development` (как код-фейл, с расходом developer-retry). Так до фикса **любая** self-hosting задача, дойдя до `deploy-staging`, была обречена на ложный откат. Аддитивно, под флагами, never-raise, скоуп self-hosting; `STAGE_TRANSITIONS` / реестр `QG_CHECKS` / семантика и имена `check_staging_status`/`_parse_staging_status` / machine-verdict-ключи (`staging_status:`/`deploy_status:`) / схема БД — **байт-в-байт не тронуты** (замена *стратегии исполнения продюсера* `15-staging-log.md`, **не** гейта/стадии; зеркало инварианта ORCH-115 NFR-1). ADR: `docs/work-items/ORCH-123/06-adr/ADR-001-host-side-staging-execution-and-env-classification.md`, сквозной `docs/architecture/adr/adr-0049-host-side-docker-execution-boundary.md`. + - **Host-side ssh-стратегия (D1):** `staging_runner.build_staging_command()` теперь обёртывает ту же `docker exec orchestrator-staging python3 … staging_check.py … --mode stub` в `ssh -o StrictHostKeyChecking=no @ ''` (зеркало `self_deploy.build_deploy_command` / `image_freshness.image_revision(ssh_target=…)`); канал — существующий доверенный (`ORCH_DEPLOY_SSH_HOST=127.0.0.1`, ssh-ключ смонтирован `:ro`, `openssh-client` в образе) → **новых секретов/привилегий не вводится** (NFR-3). Меняется **инициатор/канал** запуска, **не** сама сюита (она по-прежнему бежит **внутри** `orchestrator-staging` 8501). **Security (D2):** docker CLI/SDK в контейнер **не добавляется**, `docker.sock` **не используется изнутри** — это было бы root-эквивалентным расширением поверхности атаки (доступным и LLM-агентам); host-side ssh достигает цели без расширения привилегий. + - **Трёхсторонняя классификация исхода (D3, чистая `classify_staging_outcome`, зеркало `merge_gate.classify_retest_failure`):** `suite-ran` (распознанный exit-код, кроме 255, **без** env-маркера в stderr → доверяем коду: `0→SUCCESS`/`≠0→FAILED`; анти-over-tolerance BR-3 — реальный фейл сюиты **никогда** не реклассифицируется в инфру), `permanent-env` (spawn-error `rc=None` без таймаута / нет ssh-target / `rc∈{126,127}` / env-маркер `No such container`/`Cannot connect to the Docker daemon`/`command not found` → ретрай бессмыслен), `transient-infra` (timeout / ssh transport `rc=255` / неизвестный сигнал → ретрай осмыслен). Дизамбигуация коллизии `exit=1` (`docker exec` «No such container»=1 vs суита fail=1) — **скан stderr на env-маркеры**, не голый exit-код; fail-safe при неоднозначности → `transient-infra` (DEFER), никогда тихий `suite-ran`. + - **Маршрутизация исходов (D4) — инвариант «инфра ≠ код-фейл»:** `suite-ran` → **без изменений** (ORCH-115): write `15-staging-log.md` + `advance_stage` (FAILED → прежний откат `deploy-staging → development` + developer-retry, байт-в-байт). `permanent-env` → **немедленный infra-HOLD**: DEFER пропускается (FR-3), `15-staging-log.md` **не** пишется (нет ложного FAILED), `advance` **не** зовётся, developer-retry **не** жжётся; структурный ERROR + операторский alert «инфра/окружение, НЕ дефект кода». `transient-infra` → существующий bounded DEFER, **но на исчерпании бюджета** — тоже **infra-HOLD + alert** (СУПЕРСЕД ORCH-115 D5: прежний fail-closed `write_staging_log("FAILED") + advance` ложно откатывал не-прояснившуюся инфру как код-фейл, нарушая BR-2). Корневой инвариант: «сюита **не** исполнилась» (environment ИЛИ инфра) **никогда** не оканчивается код-фейл-откатом `→ development` и **никогда** не жжёт developer-retry — закрывает RCA-класс ORCH-110 на staging-ребре. Задача **держится** на `deploy-staging`; reconciler `advance_if_gate_passed` на red-гейте возвращает `None` (без отката, R-2 верифицирован) → оператор re-drive после починки окружения. + - **Prod-like preflight (D5):** `staging_runner.preflight()` (bounded, never-raise, self-hosting-скоуп — `applies()` первым) пробит host-side канал на **старте сервиса** в `main.lifespan` (best-effort, после `requeue_running_jobs`/`recover_on_startup`, **никогда не блокирует старт**): ssh-зонд `command -v docker && docker inspect -f '{{.State.Running}}' orchestrator-staging` распознаёт «нет docker на хосте» / «staging-контейнер не поднят» / «ssh недоступен» / «нет ssh-target» **до** того, как реальная задача дойдёт до ложного исхода. Чисто наблюдательный — не гейтит конвейер; лог + Telegram-алерт + поле в `snapshot()`. + - **Условность / обратимость (D6):** новый флаг `staging_runner_exec_host_side: bool = True` (env `ORCH_STAGING_RUNNER_EXEC_HOST_SIDE`, дефолт = боевое) — `True` → host-side ssh; `False` → прежний in-container `docker exec` (валиден лишь там, где docker CLI запечён в образ). Переиспользуются `staging_runner_enabled`/`_repos`/`_timeout_s`/`_infra_max_retries`/`_retry_delay_s` + `deploy_ssh_user`/`deploy_ssh_host`. Откат — `ORCH_STAGING_RUNNER_EXEC_HOST_SIDE=false` (in-container 1:1) или `ORCH_STAGING_RUNNER_ENABLED=false` (LLM-deployer 1:1). Наблюдаемость (D8): счётчик `permanent_env` (infra-HOLD; отличён от `failed`=код-фейл и `deferred`=транзиент) + `exec_host_side` + `preflight` в блоке `staging_runner` `GET /queue`; один структурный лог-вердикт на прогон (`outcome ∈ {code-pass,code-fail,transient-infra,permanent-env}`). + - **Документация границы исполнения (D9):** `docs/operations/INFRA.md` (новый под-раздел «Граница исполнения docker-операций — host-side») + `docs/architecture/README.md` (host-side стратегия + трёхсторонняя классификация) — зафиксировано, что **все** docker-операции self-hosting (прод-деплой ORCH-036, image-freshness ORCH-058, staging-runner ORCH-123) исполняются host-side через ssh, docker CLI в контейнере нет, `docker.sock` сознательно не используется изнутри. Покрытие — `tests/test_orch123_staging_runner_exec.py` (TC-01 — обязательный регресс ORCH-116: КРАСНЫЙ до фикса / ЗЕЛЁНЫЙ после; TC-02…TC-14 + регресс R-2 «held не становится rollback») + зелёный анти-дрейф `tests/test_orch115_staging_runner.py` (TC-14: инварианты ORCH-115 целы; 3 теста обновлены под суперсед D4/D8/D1). - **Детерминированный staging-раннер вместо LLM-деплойера на `deploy-staging`** (ORCH-115, `feat`): первый реализованный срез determinization-roadmap (ORCH-118 A6, `replace-deterministic-now`) — на стадии `deploy-staging` для self-hosting `orchestrator` **LLM-агент `deployer` заменён детерминированным кодом** (`src/staging_runner.py`). Работа агента на этой стадии была чисто механической (запуск staging-сюиты + маппинг exit-кода `staging_check.py` → `staging_status:`); каждый прогон жёг токены/время opus-агента (~40–120k / 3–15 мин) и встраивал недетерминизм LLM в точку ветвления гейта. **Инвариант (NFR-1):** это замена *продюсера* артефакта, **не** гейта — контракт `15-staging-log.md`, гейт `check_staging_status`/`_parse_staging_status`, `STAGE_TRANSITIONS`, machine-verdict `staging_status:`, схема БД — **байт-в-байт не тронуты**. Аддитивно, под kill-switch, never-raise, fail-closed, скоуп self-hosting. ADR: `docs/work-items/ORCH-115/06-adr/ADR-001-deterministic-staging-runner.md`, сквозной `docs/architecture/adr/adr-0048-deterministic-staging-runner.md`. - **Перехват в `launch_job` до `_spawn` (D1):** `if job.agent=="deployer" and staging_runner.should_intercept(job)` → `_run_staging_runner_job` (зеркало `_run_deploy_finalizer_job`, прецедент `deploy-finalizer`/`post-deploy-monitor` `launcher.py:389/394`): синхронно ведёт `jobs`-строку через `mark_job`, возвращает `None` (нет `agent_runs`-строки, нет токенов). Дискриминатор «staging vs prod» — **стадия задачи** `deploy-staging` (роль `deployer` общая для `deploy-staging`/`deploy`), не имя роли; `should_intercept` never-raise → `False` → штатный `_spawn` (fail-safe к LLM-пути). Для не-self репо `applies==False` → прод-`deployer` никогда не перехватывается. - **Leaf `src/staging_runner.py` (новый, чистый never-raise):** по образцу `self_deploy`/`proc_group`/`staging_verdict` (на импорте только `config`/`proc_group`; `db`/`git_worktree`/`stage_engine`/`qg.checks`/`notifications` — лениво). Исполняет ту же сюиту (`docker exec orchestrator-staging python3 //scripts/staging_check.py --base-url http://localhost: --mode stub`, арги из config — ORCH-101, без host-хардкодов) через `proc_group.run_in_process_group` (tree-kill, таймаут `staging_runner_timeout_s=600`, малформ/непозитив → дефолт + WARNING); маппит exit-код **единым** контрактом `self_deploy.map_exit_code_to_status` (`0→SUCCESS`/иначе/None→`FAILED`, fail-closed; ORCH-061 infra-tolerance остаётся внутри `staging_check.py`, раннер не пересуживает — BR-4); пишет `15-staging-log.md` (тот же machine-key `staging_status:` UPPERCASE + 52c-схема, `author_agent: staging-runner`/`model_used: n/a`) + best-effort commit/push в **фичеветку** (не в `main` — гейт читает worktree первым, усиливает AC-8/BR-7); вызывает **существующий** `advance_stage(current_stage="deploy-staging", finished_agent="deployer")` — без новых рёбер/исходов (transition-lease ORCH-114 берётся внутри `advance_stage`, раннер не трогает — граница O1). diff --git a/CLAUDE.md b/CLAUDE.md index 2d5ac85..62b1fde 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -456,6 +456,42 @@ Plane, **не** Quality Gate и **не** стадия). `tests/test_orch115_staging_runner.py` (TC-01…TC-13) + зелёные LLM-анти-дрейф тесты (TC-14). Детали — `docs/work-items/ORCH-115/06-adr/ADR-001-deterministic-staging-runner.md`, сквозной `docs/architecture/adr/adr-0048-deterministic-staging-runner.md`. +- **ORCH-123 (host-side исполнение + классификация environment-дефекта, bug→escalate full-cycle, + adr-0049):** фикс инцидента **ORCH-116** — раннер ORCH-115 вызывал `docker exec` **изнутри** + прод-контейнера, где **нет docker CLI** (`Dockerfile:11` несёт только `openssh-client git curl`; + `docker.sock` смонтирован, клиента нет) → `FileNotFoundError` → постоянный environment-дефект + ложно маршрутизировался как код-фейл-откат `deploy-staging → development` (с расходом + developer-retry); любая self-hosting задача на `deploy-staging` была обречена. Аддитивно, под + флагами, never-raise, скоуп self-hosting; `STAGE_TRANSITIONS`/реестр `QG_CHECKS`/семантика и имена + `check_staging_status`/`_parse_staging_status`/machine-verdict-ключи/схема БД — **байт-в-байт не + тронуты** (замена *стратегии исполнения продюсера*, **не** гейта; зеркало ORCH-115 NFR-1). + **(D1)** `build_staging_command` обёртывает ту же `docker exec orchestrator-staging … staging_check.py + … --mode stub` в `ssh -o StrictHostKeyChecking=no @ '<…>'` + (зеркало `self_deploy`/`image_freshness`; канал доверенный `ORCH_DEPLOY_SSH_HOST=127.0.0.1`, + ssh-ключ `:ro`, `openssh-client` в образе — новых секретов/привилегий нет; сюита по-прежнему бежит + **внутри** `orchestrator-staging` 8501, меняется лишь инициатор `docker exec`). **(D2 security)** + docker CLI/SDK в контейнер **не** добавляется, `docker.sock` **не** используется изнутри (это + root-эквивалентное расширение поверхности атаки, доступное и LLM-агентам). **(D3)** двухуровневый + исход ORCH-115 заменён **трёхсторонней** чистой `classify_staging_outcome` (зеркало + `merge_gate.classify_retest_failure`): `suite-ran` (распознанный exit-код ≠255 **без** env-маркера в + stderr → доверяем коду `0→SUCCESS`/`≠0→FAILED`; анти-over-tolerance BR-3), `permanent-env` + (spawn-error `rc=None`/нет ssh-target/`rc∈{126,127}`/env-маркер `No such container`/`Cannot connect + to the Docker daemon` → ретрай бессмыслен), `transient-infra` (timeout/ssh `rc=255`/неизвестно → + ретрай осмыслен); дизамбигуация `exit=1`-коллизии — скан stderr на env-маркеры, не голый код; + fail-safe → `transient-infra`. **(D4 инвариант «инфра ≠ код-фейл»)** `permanent-env` → немедленный + **infra-HOLD** (DEFER пропускается, `15-staging-log.md` НЕ пишется, advance НЕ зовётся, developer-retry + НЕ жжётся, alert «НЕ дефект кода»); `transient-infra` → bounded DEFER, на исчерпании — тоже + infra-HOLD+alert (**супершид ORCH-115 D5** fail-closed-FAILED-отката). «Сюита **не** исполнилась» + (env ИЛИ инфра) **никогда** не оканчивается код-фейл-откатом — закрывает RCA-класс ORCH-110 на + staging-ребре; задача держится на `deploy-staging` (reconciler `advance_if_gate_passed` на red-гейте + → `None`, без отката, R-2). **(D5)** `preflight()` пробит host-side канал на старте `main.lifespan` + (best-effort, never-blocks). **(D6)** новый флаг `staging_runner_exec_host_side=True` + (env `ORCH_STAGING_RUNNER_EXEC_HOST_SIDE`); откат — `=false` (in-container 1:1) или + `ORCH_STAGING_RUNNER_ENABLED=false` (LLM-deployer 1:1). **(D8)** счётчик `permanent_env` + `exec_host_side` + + `preflight` в блоке `staging_runner` `GET /queue`. Покрытие — `tests/test_orch123_staging_runner_exec.py` + (TC-01 обязательный регресс ORCH-116 red→green; TC-02…TC-14 + R-2) + зелёный анти-дрейф ORCH-115. + Детали — `docs/work-items/ORCH-123/06-adr/ADR-001-host-side-staging-execution-and-env-classification.md`, + сквозной `docs/architecture/adr/adr-0049-host-side-docker-execution-boundary.md`. ## Машинный журнал уроков (ORCH-098) Шаг 1 («Фундамент», F2) эпика саморазвития: формализует свободнотекстовые «уроки» из `memory/` в diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 32e54ba..d52f5b9 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -10,7 +10,7 @@ - **Review/Test Parsers** (`src/review_parse.py`, ORCH-046) — defensive-извлечение дословного must-fix текста из артефактов для встраивания в `task_desc` заворота: `extract_review_findings` (P0/P1 из `12-review.md`), `extract_test_failures` (фрагмент тела `13-test-report.md`). Контракт «never raise»: любая ошибка → `""`. - **Quality Gates** (`src/qg/checks.py`) — проверки выхода со стадии, реестр `QG_CHECKS`. - **Agent Launcher** (`src/agents/launcher.py`) — запуск Claude CLI агентов в изолированном git worktree, мониторинг, auto-advance. Модель/эффорт каждого агента резолвятся из config (`resolve_agent_model`/`resolve_agent_effort`, ORCH-41), а не из frontmatter промпта. **ORCH-74:** имя модели валидируется форматом `^claude-…$` (`is_valid_model`) перед `--model`; невалидное → лог + откат на следующий уровень/CLI-дефолт (never-break, как `VALID_EFFORTS` для эффорта). Тот же предикат гардит inline-чтение `--fallback-model`. **ORCH-109 ([adr-0040](adr/adr-0040-agent-timeout-budgets-and-launch-model-stamp.md)):** (1) резолвенная **модель стампится в `agent_runs.model` в момент launch** (`_spawn`, объединённый `UPDATE … SET model=?, effort=?` рядом со стампом эффорта ORCH-087; пустой резолв → `NULL`; never-raise) → модель видна не-`null` при любом исходе прогона, включая timeout-kill (`exit_code=-9`), и in-flight в `GET /metrics`/`GET /queue` (`get_running_agents` уже отдаёт `model`); постфактум `record_usage` (`model=COALESCE(?, model)`) остаётся **обогащением**, не единственным источником истины. (2) **Per-role wall-clock бюджеты** через выделенные ключи `agent_timeout_developer_s=3600`/`agent_timeout_reviewer_s=3000` (лестница `_resolve_timeout`: `agent_timeout_overrides_json` → выделенный ключ роли → `agent_timeout_seconds=1800`; прочие роли — байт-в-байт; малформный/вне-диапазонный конфиг → дефолт + WARNING). Инвариант reaper ORCH-065 сохранён синхронным поднятием `reaper_max_running_s` 3600→**5400** (`5400 > max(timeout)3600 + grace20`). FR-5 анти-salvage — структурно: продвижение гейтится `if exit_code==0`, timeout-kill → `_finalize_job` (retry/fail), не advance. `STAGE_TRANSITIONS`/`QG_CHECKS`/схема БД не тронуты. -- **Staging-runner** (`src/staging_runner.py`, ORCH-115 — [adr-0048](adr/adr-0048-deterministic-staging-runner.md)) — чистый **never-raise** leaf (паттерн `self_deploy`/`proc_group`/`staging_verdict`), заменяющий **LLM-агента `deployer` на стадии `deploy-staging`** детерминированным кодом (первый реализованный срез determinization-roadmap, A6/`replace-deterministic-now`). Перехват в `launch_job` **до `_spawn`** (рядом с D1/D2 `deploy-finalizer`/`post-deploy-monitor`): дискриминатор — **стадия задачи** `deploy-staging` **И** `applies(repo)` (kill-switch `staging_runner_enabled` + скоуп `staging_runner_repos`, пусто → self-hosting only), `should_intercept` never-raise → `False` → штатный `_spawn` (fail-safe к LLM). Раннер (зеркало `run_deploy_finalizer`) исполняет ту же staging-сюиту (`docker exec orchestrator-staging … staging_check.py`) через `proc_group` (tree-kill, таймаут `staging_runner_timeout_s=600`), маппит exit-код единым контрактом `self_deploy.map_exit_code_to_status` (`0→SUCCESS`/иначе→`FAILED`; ORCH-061 infra-tolerance остаётся внутри `staging_check.py`), пишет `15-staging-log.md` (тот же machine-key `staging_status:`, `author_agent: staging-runner`/`model_used: n/a`) + best-effort push в фичеветку, и вызывает **существующий** `advance_stage(current_stage="deploy-staging", finished_agent="deployer")` — без новых рёбер/исходов; transition-lease ORCH-114 берётся внутри `advance_stage` (раннер не трогает). **Двухуровневый исход (анти-ORCH-110):** сюита исполнилась → verdict→advance (FAILED → тот же откат `deploy-staging → development` + developer-retry, что у LLM); tool-error/таймаут → bounded defer (`staging_runner_infra_max_retries`) → fail-closed `FAILED` + alert на исчерпании (инфра ≠ код-фейл, никогда тихий advance/ложный green). `STAGE_TRANSITIONS`/`QG_CHECKS`/`check_staging_status`/`_parse_staging_status`/machine-verdict/схема БД — **байт-в-байт не тронуты** (замена *продюсера*, не гейта). Наблюдаемость — in-process счётчики + блок `staging_runner` в `GET /queue`. Откат — `ORCH_STAGING_RUNNER_ENABLED=false` (прежний LLM-deployer-путь байт-в-байт). Детали — `docs/work-items/ORCH-115/06-adr/ADR-001-deterministic-staging-runner.md`. +- **Staging-runner** (`src/staging_runner.py`, ORCH-115 — [adr-0048](adr/adr-0048-deterministic-staging-runner.md)) — чистый **never-raise** leaf (паттерн `self_deploy`/`proc_group`/`staging_verdict`), заменяющий **LLM-агента `deployer` на стадии `deploy-staging`** детерминированным кодом (первый реализованный срез determinization-roadmap, A6/`replace-deterministic-now`). Перехват в `launch_job` **до `_spawn`** (рядом с D1/D2 `deploy-finalizer`/`post-deploy-monitor`): дискриминатор — **стадия задачи** `deploy-staging` **И** `applies(repo)` (kill-switch `staging_runner_enabled` + скоуп `staging_runner_repos`, пусто → self-hosting only), `should_intercept` never-raise → `False` → штатный `_spawn` (fail-safe к LLM). Раннер (зеркало `run_deploy_finalizer`) исполняет ту же staging-сюиту (`docker exec orchestrator-staging … staging_check.py`) через `proc_group` (tree-kill, таймаут `staging_runner_timeout_s=600`), маппит exit-код единым контрактом `self_deploy.map_exit_code_to_status` (`0→SUCCESS`/иначе→`FAILED`; ORCH-061 infra-tolerance остаётся внутри `staging_check.py`), пишет `15-staging-log.md` (тот же machine-key `staging_status:`, `author_agent: staging-runner`/`model_used: n/a`) + best-effort push в фичеветку, и вызывает **существующий** `advance_stage(current_stage="deploy-staging", finished_agent="deployer")` — без новых рёбер/исходов; transition-lease ORCH-114 берётся внутри `advance_stage` (раннер не трогает). **Двухуровневый исход (анти-ORCH-110):** сюита исполнилась → verdict→advance (FAILED → тот же откат `deploy-staging → development` + developer-retry, что у LLM); tool-error/таймаут → bounded defer (`staging_runner_infra_max_retries`) → fail-closed `FAILED` + alert на исчерпании (инфра ≠ код-фейл, никогда тихий advance/ложный green). `STAGE_TRANSITIONS`/`QG_CHECKS`/`check_staging_status`/`_parse_staging_status`/machine-verdict/схема БД — **байт-в-байт не тронуты** (замена *продюсера*, не гейта). Наблюдаемость — in-process счётчики + блок `staging_runner` в `GET /queue`. Откат — `ORCH_STAGING_RUNNER_ENABLED=false` (прежний LLM-deployer-путь байт-в-байт). Детали — `docs/work-items/ORCH-115/06-adr/ADR-001-deterministic-staging-runner.md`. **ORCH-123 (фикс стратегии исполнения — [adr-0049](adr/adr-0049-host-side-docker-execution-boundary.md)):** раннер ORCH-115 вызывал `docker exec` **изнутри** прод-контейнера, где **нет docker CLI** (образ несёт только `openssh-client git curl`, `Dockerfile:11`) → `FileNotFoundError` → постоянный environment-дефект ложно откатывался как код-фейл `deploy-staging → development` (инцидент ORCH-116). Фикс: `build_staging_command` теперь исполняет ту же `docker exec … staging_check.py … --mode stub` **host-side** через доверенный ssh-канал `ssh ''` (зеркало `self_deploy`/`image_freshness`, флаг `ORCH_STAGING_RUNNER_EXEC_HOST_SIDE=true`, дефолт; меняется **инициатор/канал**, не сама сюита — она по-прежнему бежит внутри `orchestrator-staging` 8501). Двухуровневый исход ORCH-115 заменён **трёхсторонней классификацией** (`classify_staging_outcome`): `suite-ran` (распознанный exit-код без env-маркера в stderr → доверяем коду: `0→SUCCESS`/`≠0→FAILED`+прежний откат, анти-over-tolerance), `permanent-env` (spawn-error/нет ssh-target/126-127/env-маркер `No such container`/`Cannot connect to the Docker daemon` → **немедленный infra-HOLD**: без отката, без developer-retry, DEFER пропускается), `transient-infra` (timeout/ssh 255 → bounded DEFER, на исчерпании → infra-HOLD, **не** прежний fail-closed FAILED+откат). Корневой инвариант: «сюита **не** исполнилась» (environment ИЛИ инфра) **никогда** не оканчивается код-фейл-откатом — закрывает RCA-класс ORCH-110 на staging-ребре. Plus prod-like preflight host-side канала в `main.lifespan` (best-effort, never-blocks). `STAGE_TRANSITIONS`/`QG_CHECKS`/`check_staging_status`/machine-verdict/схема БД — байт-в-байт не тронуты. Детали — `docs/work-items/ORCH-123/06-adr/ADR-001-host-side-staging-execution-and-env-classification.md`. - **Queue** (`src/queue_worker.py`, ORCH-1) — персистентная очередь задач (SQLite `jobs`), atomic claim, max_concurrency, ретраи, restart-safe. **ORCH-026:** `claim_next_job` гейтит задачи с незавершёнными зависимостями (`job_deps`, `NOT EXISTS`) без занятия слота; декларации/циклы — leaf `src/task_deps.py`. - **Job-reaper** (`src/job_reaper.py`, ORCH-065 — [adr-0011](adr/adr-0011-job-reaper-lease-reclaim.md)) — фоновый daemon-поток (каркас `reconciler`), стартует/останавливается в `main.lifespan` (после `reconciler.start()` / перед `worker.stop()`). Детектирует «мёртвый» `running`-job **без рестарта** процесса (Tier-1 мёртвый `jobs.pid` после `reaper_dead_ticks` тиков; Tier-2 `agent_runs.exit_code` записан, а job ещё `running`; Tier-3 backstop `reaper_max_running_s`) и приводит строку к корректному статусу через те же контракты (`_try_advance_stage`/`_finalize_job`, gate-driven; exit≠0/неизвестно → `attempts`. `STAGE_TRANSITIONS`/`QG_CHECKS`/`check_*`/machine-verdict/схемы существующих таблиц — байт-в-байт. Подробнее ниже (§ «Единое владение side-effectful переходами»). Детали — `docs/work-items/ORCH-114/06-adr/ADR-001-transition-ownership-lease-and-stage-cas.md`. diff --git a/docs/operations/INFRA.md b/docs/operations/INFRA.md index c0efd19..eed9c3d 100644 --- a/docs/operations/INFRA.md +++ b/docs/operations/INFRA.md @@ -244,10 +244,39 @@ watchdog'а: **watchdog сигналит, pruner убирает**. - **Свежесть staging-образа (ORCH-058):** на ребре `deploy-staging → deploy` (ПОСЛЕ merge-gate, ДО Phase A) QG-под-чек `check_staging_image_fresh` пересобирает staging-образ из валидированного коммита и пересоздаёт 8501 (Strategy A), а хук перед build-once retag fail-closed сверяет OCI-лейбл `revision` с `EXPECTED_REVISION` (Strategy B). Гарантирует: в прод промоутится РОВНО провалидированный артефакт (инцидент LESSONS_ORCH-036 п.4 — тихий промоут устаревшего образа). Сборки/recreate — ТОЛЬКО staging (8501); FAIL → откат на `development`. Условный: реален только для self-hosting. - **Гигиена shared deploy-базы (ORCH-112):** self-deploy `git pull origin main` устойчив к грязному рабочему дереву deploy-базы (модифицированные tracked + untracked-остатки failed/cancelled/брошенных задач). Хук `--deploy` перед pull приводит базу к чистому `origin/main` (resilient-pull: `git fetch` + `git reset --hard origin/main` + `git clean -fd`), **строго сохраняя** rollback-снимки `.deploy-prev-image-*`, `deploy-hook.log`, gitignored `.env`/`data/`/`*.db` (НИКОГДА `-x`!), sibling `.deploy-state-*`/`.merge-lease-*.json`, `.git/worktrees/*`. Гейт — kill-switch `ORCH_CHECKOUT_HYGIENE_ENABLED` (дефолт `True`; off → голый pull 1:1); скоуп `ORCH_CHECKOUT_HYGIENE_REPOS` (пусто → self-hosting only). Грязь базы детектируется → лог + Telegram-алерт (Phase-C finalizer). Решает инцидент ORCH-111 (грязь ORCH-104 заблокировала `git pull`). Детально — `docs/work-items/ORCH-112/06-adr/ADR-001`, сквозной adr-0044. +### Граница исполнения docker-операций — host-side, НЕ изнутри прод-контейнера (ORCH-123) + +**Инвариант (нормативно, adr-0049):** прод-контейнер `orchestrator` несёт **только** +`openssh-client git curl` (+ pinned gitleaks) — **бинаря `docker` в образе НЕТ** (`Dockerfile:11`, +`python:3.12-slim` его не несёт). `/var/run/docker.sock` **смонтирован** (`docker-compose.yml`, +rw + `group_add 999`, «МИНА 1» ORCH-040), но **сознательно НЕ используется изнутри** контейнера: нет +docker-клиента, и добавлять его (CLI/SDK) **нельзя** — это активировало бы дремлющий +root-эквивалентный путь для всего, что бежит в контейнере, включая LLM-агентов (security-разбор — +ADR-001 D2 / adr-0049 / R-1). + +Поэтому **все** docker-операции self-hosting исполняются **host-side** через **ssh на `127.0.0.1`** +(`ORCH_DEPLOY_SSH_HOST`/`ORCH_DEPLOY_SSH_USER`, ssh-ключ смонтирован `:ro`), где docker CLI есть: +- **прод-деплой** (ORCH-036, `self_deploy.build_deploy_command`) — `ssh … setsid bash hook --deploy`; +- **image-freshness** (ORCH-058, `image_freshness.image_revision`/`rebuild_staging_image`) — `ssh … docker …`; +- **staging-runner** (ORCH-123, `staging_runner.build_staging_command`) — `ssh … docker exec orchestrator-staging python3 … staging_check.py … --mode stub`. + +Сама staging-сюита (`scripts/staging_check.py`) **по-прежнему** исполняется **внутри** контейнера +`orchestrator-staging` (8501) — меняется лишь **кто инициирует** `docker exec` (хост через ssh, а не +прод-контейнер). До ORCH-123 staging-runner (ORCH-115) вызывал `docker exec` **изнутри** +прод-контейнера → `FileNotFoundError` (нет `docker`) → постоянный environment-дефект ложно +маршрутизировался как код-фейл-откат `deploy-staging → development` (инцидент ORCH-116). Фикс: +host-side ssh-обёртка (флаг `ORCH_STAGING_RUNNER_EXEC_HOST_SIDE=true`, дефолт) + трёхсторонняя +классификация (suite-ran / permanent-env / transient-infra), где environment/инфра **никогда** не +оканчивается код-фейл-откатом (infra-HOLD + алерт «инфра, НЕ дефект кода»), и prod-like preflight +канала на старте сервиса. Откат — `ORCH_STAGING_RUNNER_EXEC_HOST_SIDE=false` (прежний in-container +вызов — валиден лишь там, где docker CLI запечён в образ) или `ORCH_STAGING_RUNNER_ENABLED=false` +(LLM-deployer 1:1). Детали — `docs/work-items/ORCH-123/06-adr/ADR-001`, сквозной adr-0049. + **Правила для агентов при задачах ORCH:** 1. НЕ перезапускать / не ронять прод-контейнер `orchestrator` в рамках задачи. 2. Все проверки деплоя — на staging (8501), боевой 8500 не трогать. 3. Деплой self — только через хук с health-check + авто-rollback (`DEPLOY_HOOK.md`). +4. docker-операции исполняются **host-side через ssh** — в контейнере `docker` CLI нет (ORCH-123). ## Эксплуатация (быстрые команды) ```bash diff --git a/src/main.py b/src/main.py index 3877bbd..9655e91 100644 --- a/src/main.py +++ b/src/main.py @@ -79,6 +79,21 @@ async def lifespan(app: FastAPI): except Exception as e: log.warning(f"Transition-lease recovery skipped: {e}") + # ORCH-123 (adr-0049 / D5 / FR-4): prod-like preflight of the host-side staging + # execution channel. The deploy-staging staging-runner (ORCH-115) runs the suite + # HOST-SIDE over ssh because the prod container ships no docker CLI (Dockerfile:11); + # probe the channel at startup so a broken environment (no docker on host / staging + # down / ssh unreachable / no ssh target) surfaces HERE — not postfactum as a false + # rollback of a real task (incident ORCH-116). Purely observational: it never blocks + # the start / gates the pipeline. never raises (runs after requeue + lease-recovery). + try: + from . import staging_runner + ok, reason = staging_runner.preflight() + if not ok: + log.warning(f"Staging-runner preflight: {reason}") + except Exception as e: + log.warning(f"Staging-runner preflight skipped: {e}") + # ORCH-065: proactive startup reclaim of dead/stale merge-leases, next to the # queue-recovery above. A lease held by the previous (now dead) process pid is # released at once instead of waiting for the TTL / a foreign acquire so the diff --git a/src/staging_runner.py b/src/staging_runner.py index 2b9f011..a4c1fce 100644 --- a/src/staging_runner.py +++ b/src/staging_runner.py @@ -178,26 +178,74 @@ def should_intercept(job: dict) -> bool: # --------------------------------------------------------------------------- # Suite execution (D3 / FR-2 / NFR-3 / AC-8 / AC-9) # --------------------------------------------------------------------------- -def build_staging_command() -> list[str]: - """Build the canonical staging-suite argv (same command the LLM-deployer ran). +def _ssh_target() -> str: + """ssh ``user@host`` for host-side execution, or ``""`` when no host is + configured (mirror ``self_deploy``/``image_freshness._ssh_target``). On the prod + host ``ORCH_DEPLOY_SSH_HOST=127.0.0.1`` is set by compose; the config default is + empty so tests / non-self contexts fall back to the in-container command.""" + host = (settings.deploy_ssh_host or "").strip() + if not host: + return "" + user = (settings.deploy_ssh_user or "").strip() + return f"{user}@{host}" if user else host - ``docker exec python3 //scripts/staging_check.py - --base-url http://localhost: --mode stub``. Host-specifics come from - config (ORCH-101, no host hardcodes). Self-hosting safety (BR-7 / AC-8 / TC-12): - NO restart of 8500, NO ``docker compose up orchestrator`` / ``--build``, NO - force-push, NO ``.env`` edit — the runner only reads/executes the staging suite - (8501) and writes a log. + +def _channel_ssh_configured() -> bool: + """Whether the execution channel is viable w.r.t. ssh config (fed to + ``classify_staging_outcome``). In-container mode (host-side disabled) is always + 'configured' — its viability is decided purely by the result signals; host-side + mode needs a non-empty ssh target (R-6: an empty target -> ``permanent-env``). + never-raise -> assume configured (rely on the result signals).""" + try: + if not bool(getattr(settings, "staging_runner_exec_host_side", True)): + return True + return bool(_ssh_target()) + except Exception: # noqa: BLE001 - never-raise + return True + + +def build_staging_command() -> list[str]: + """Build the staging-suite argv (same suite command the LLM-deployer ran). + + The INNER command is unchanged: ``docker exec python3 + //scripts/staging_check.py --base-url + http://localhost: --mode stub``. Host-specifics come from config + (ORCH-101, no host hardcodes). + + ORCH-123 (D1, adr-0049): ``docker`` lives on the HOST — the prod container ships + only ``openssh-client git curl`` (``Dockerfile:11``), so a ``docker exec`` spawned + FROM INSIDE the prod container hit ``FileNotFoundError`` (incident ORCH-116). When + ``staging_runner_exec_host_side`` (default True) is set AND an ssh target is + configured, the inner command is wrapped in ``ssh ''`` + so it runs host-side over the existing trusted ssh channel (mirror + ``self_deploy.build_deploy_command`` / ``image_freshness.image_revision``). With + the flag off OR no ssh target configured (tests / non-self contexts) it falls back + to the prior in-container ``docker exec`` (valid only where a docker CLI is baked + into the image). + + Self-hosting safety (BR-7 / AC-8 / TC-08): the argv carries ONLY ``docker exec + python3 staging_check.py ... --mode stub`` — NO restart of 8500, + NO ``docker compose up orchestrator`` / ``--build``, NO force-push, NO ``.env`` + edit. The runner only reads/executes the staging suite (8501) and writes a log. """ from .qg.checks import SELF_HOSTING_REPO repos_dir = (settings.repos_dir or "/repos").rstrip("/") script = f"{repos_dir}/{SELF_HOSTING_REPO}/scripts/staging_check.py" base_url = f"http://localhost:{int(settings.staging_port)}" - return [ + inner_argv = [ "docker", "exec", STAGING_SERVICE, "python3", script, "--base-url", base_url, "--mode", "stub", ] + # ORCH-123 (D1): host-side ssh-wrap when enabled AND a target is configured. + if bool(getattr(settings, "staging_runner_exec_host_side", True)): + target = _ssh_target() + if target: + remote = " ".join(shlex.quote(a) for a in inner_argv) + return ["ssh", "-o", "StrictHostKeyChecking=no", target, remote] + # Fallback: prior in-container docker exec (no ssh target / flag off). + return inner_argv def _resolve_timeout() -> int: @@ -255,6 +303,71 @@ def map_exit_code_to_status(exit_code) -> str: return _map(exit_code) +# --------------------------------------------------------------------------- +# Three-way outcome classification (D3 / FR-2 / FR-3 / AC-3, AC-4, AC-6) +# --------------------------------------------------------------------------- +def classify_staging_outcome(result, ssh_configured: bool) -> str: + """Classify a staging-suite ProcResult into one of three classes (D3). Pure, + never-raise (mirror of ``merge_gate.classify_retest_failure``, ORCH-110 D2): + + * ``"suite-ran"`` — a recognised executor exit-code (any int except the + ssh transport code 255) AND no environment marker in + stderr: the suite DEMONSTRABLY executed -> trust the + code (``0->SUCCESS``, ``!=0->FAILED``). Anti-over- + tolerance (BR-3): a real suite fail is NEVER + reclassified as infra/env. + * ``"permanent-env"`` — a deterministic PERMANENT environment defect: an + stderr env-marker (no docker / no such container / + daemon unreachable), a shell "command not found / + cannot execute" code (126/127), no host-side ssh + target configured, or a bare local spawn-error + (``returncode is None`` and not a timeout). Retrying is + pointless (FR-3) -> immediate distinguishable + infra-HOLD (no rollback, no developer-retry). + * ``"transient-infra"`` — a timeout OR an ssh transport/connection failure (255) + OR any unknown signal: a retry is meaningful -> the + bounded DEFER. + + The exit=1 collision (``docker exec`` "No such container"=1 vs a real suite fail=1) + is disambiguated by SCANNING stderr for env-markers, NEVER by the bare exit-code. + Fail-safe on doubt -> ``"transient-infra"`` (DEFER), never a silent ``"suite-ran"`` + (a mis-routed environment->code-fail is exactly the defect this fixes; an over- + tolerated code-fail is guarded by trusting recognised suite exit-codes first). + """ + try: + rc = getattr(result, "returncode", None) + timed_out = bool(getattr(result, "timed_out", False)) + stderr = (getattr(result, "stderr", "") or "").lower() + + # 1. env-marker in stderr -> permanent, REGARDLESS of the exit-code (this is + # what disambiguates a `docker exec` "No such container" exit=1 from a real + # suite fail=1 — R-3). + if any(m in stderr for m in _ENV_MARKERS): + return "permanent-env" + # 2. shell "command not found" (127) / "cannot execute" (126) -> permanent. + if rc in (126, 127): + return "permanent-env" + # 3. a recognised executor exit-code (any int except the ssh transport code + # 255) WITHOUT an env-marker -> the suite executed; trust it (BR-3). This is + # checked BEFORE the channel guards so a real suite fail is never masked. + if isinstance(rc, int) and rc != 255: + return "suite-ran" + # --- below this line the suite did NOT produce a trustworthy exit-code --- + # 4. timeout / ssh transport failure (255) -> transient: a retry is meaningful. + if timed_out or rc == 255: + return "transient-infra" + # 5. host-side ssh target missing (R-6) OR a bare local spawn-error (proc_group + # degraded an OSError to rc None without a timeout) -> permanent env defect. + if not ssh_configured: + return "permanent-env" + if rc is None: + return "permanent-env" + # 6. unknown signal -> fail-safe DEFER (never a silent suite-ran). + return "transient-infra" + except Exception: # noqa: BLE001 - never-raise; unknown -> transient (no false rollback) + return "transient-infra" + + # --------------------------------------------------------------------------- # Artifact 15-staging-log.md (D6 / FR-4 / AC-2 / AC-8) — mirror write_deploy_log # --------------------------------------------------------------------------- @@ -439,17 +552,21 @@ def _infra_retry_count(task_id) -> int: return 0 -def _handle_tool_error( +def _handle_transient_infra( task_id, repo: str, work_item_id: str, branch: str, result: proc_group.ProcResult ) -> None: - """Suite did NOT execute (tool-error) -> bounded DEFER, then fail-closed (D5). + """Transient infra (timeout / ssh transport) — the suite did not execute but a + retry is meaningful (D4). Bounded DEFER: re-queue a fresh ``deployer`` job (which + re-enters this runner) with a delay + a restart-safe marker, instead of an + immediate rollback that would burn a developer-retry. - Anti ORCH-110: an infra fault is NOT a code fault, so we re-queue a fresh - ``deployer`` job (which re-enters this runner) with a delay instead of an - immediate FAILED-rollback that would burn a developer-retry. On budget exhaustion - -> write ``staging_status: FAILED`` + advance (the existing rollback) + an - INFRA-specific alert (explicitly "not a code defect"). Never a silent advance / - false green; never wedges the queue. never-raise.""" + On budget exhaustion -> **infra-HOLD + alert** (D4, SUPERSEDES ORCH-115 D5): NOT + the prior fail-closed ``write_staging_log("FAILED") + advance``, which falsely + rolled an unresolved infra hiccup back to ``development`` as a code-fail (BR-2, + anti-pattern ORCH-110). The task is HELD on ``deploy-staging`` (a red/missing gate + keeps the reconciler's ``advance_if_gate_passed`` from rolling it back — R-2); the + operator re-drives it after fixing the stand. Never a silent advance / false green; + never wedges the queue. never-raise.""" retries = _infra_retry_count(task_id) try: max_retries = int(settings.staging_runner_infra_max_retries) @@ -462,7 +579,7 @@ def _handle_tool_error( if retries < max_retries: _bump("deferred") - reason = "timeout" if result.timed_out else "suite did not execute (tool-error)" + reason = "timeout" if result.timed_out else "ssh transport/connection error" task_desc = ( f"Work item: {work_item_id}\nRepo: {repo}\nBranch: {branch}\n" f"Stage: deploy-staging\nNote: {_INFRA_RETRY_MARKER} " @@ -474,40 +591,77 @@ def _handle_tool_error( "deployer", repo, task_desc, task_id=task_id, available_at_delay_s=delay, ) logger.warning( - "Task %s (%s): staging suite did not execute (%s) -> infra-DEFER " - "(job_id=%s, attempt %d/%d)", + "Task %s (%s): staging suite did not execute (%s) -> transient-infra " + "DEFER (job_id=%s, attempt %d/%d)", task_id, work_item_id, reason, new_job, retries + 1, max_retries, ) except Exception as e: # noqa: BLE001 - never-raise logger.error("staging_runner: infra-DEFER enqueue failed for %s: %s", task_id, e) return - # Budget exhausted -> fail-closed FAILED (terminal, never a false green). - _bump("failed") + # Budget exhausted -> infra-HOLD (D4): NO write_staging_log("FAILED"), NO advance, + # NO developer-retry. The task is held on deploy-staging for the operator. + _bump("permanent_env") logger.error( - "Task %s (%s): staging tool-error DEFER budget exhausted (%d) -> fail-closed FAILED", + "Task %s (%s): staging transient-infra DEFER budget exhausted (%d) -> " + "infra-HOLD on deploy-staging (NOT a code defect, no rollback)", task_id, work_item_id, max_retries, ) - write_staging_log(repo, work_item_id, branch, result.returncode, "FAILED", - result.stdout, tool_error=True) _alert_infra_exhausted(work_item_id, max_retries) - _advance(task_id, repo, work_item_id, branch) + + +def _handle_permanent_env( + task_id, repo: str, work_item_id: str, branch: str, result: proc_group.ProcResult +) -> None: + """Permanent environment defect (D4): an immediate, distinguishable infra-HOLD. + + The DEFER cycle is SKIPPED — retrying a permanent defect (no docker / no ssh + target / container down) is pointless (FR-3). We do NOT write a FAILED staging-log, + do NOT advance, do NOT burn a developer-retry — the task is HELD on + ``deploy-staging``: a red/missing ``check_staging_status`` keeps the reconciler's + ``advance_if_gate_passed`` from rolling it back to ``development`` (R-2), and the + rollback-to-development path (``advance_stage(finished_agent="deployer")``) is never + taken. A structured error log + an operator alert ("infra/environment, NOT a code + defect") make the hold visible; the operator re-drives after fixing the + environment. never-raise.""" + detail = (getattr(result, "stderr", "") or "").strip()[:300] + logger.error( + "Task %s (%s): PERMANENT staging environment defect -> infra-HOLD on " + "deploy-staging (NOT a code defect, no rollback, no developer-retry): %s", + task_id, work_item_id, detail or "", + ) + _alert_permanent_env(work_item_id, detail) def _alert_infra_exhausted(work_item_id: str, max_retries: int) -> None: """Best-effort Telegram alert that the staging suite never executed (infra, NOT a - code defect) after the retry budget. never-raise.""" + code defect) after the retry budget -> infra-HOLD (no rollback). never-raise.""" try: from .notifications import send_telegram, link_for send_telegram( - f"\U0001f6a8 {link_for(work_item_id)}: staging suite не запустилась " - f"(инфра, НЕ дефект кода) после {max_retries} попыток — fail-closed FAILED, " - f"откат на development. Нужно проверить staging-стенд." + f"\U0001f6a8 {link_for(work_item_id)}: staging-сюита не запустилась " + f"(инфра, НЕ дефект кода) после {max_retries} попыток — задача удержана на " + f"deploy-staging (без отката на development). Нужно проверить staging-стенд." ) except Exception as e: # noqa: BLE001 - never-raise logger.warning("staging_runner: infra-exhausted alert failed for %s: %s", work_item_id, e) +def _alert_permanent_env(work_item_id: str, detail: str) -> None: + """Best-effort Telegram alert that the staging suite could not execute due to a + permanent environment defect (NOT a code defect) -> infra-HOLD. never-raise.""" + try: + from .notifications import send_telegram, link_for + tail = f": {detail}" if detail else "" + send_telegram( + f"\U0001f6a8 {link_for(work_item_id)}: staging-сюита не смогла исполниться — " + f"постоянный дефект окружения (инфра, НЕ дефект кода){tail}. Задача удержана " + f"на deploy-staging до починки окружения (без отката на development)." + ) + except Exception as e: # noqa: BLE001 - never-raise + logger.warning("staging_runner: permanent-env alert failed for %s: %s", work_item_id, e) + + # --------------------------------------------------------------------------- # Entry point (D2) — owns the full deterministic flow, mirror run_deploy_finalizer # --------------------------------------------------------------------------- @@ -517,10 +671,12 @@ def run_staging_gate(job: dict) -> None: Flow (mirror of ``stage_engine.run_deploy_finalizer``): 1. resolve ``work_item_id`` / ``branch`` by ``task_id``; 2. execute the staging suite (D3) -> ProcResult; - 3. suite EXECUTED -> map exit-code -> ``staging_status:``, write - ``15-staging-log.md``, initiate the existing gate via ``advance_stage`` (D7); - 4. suite did NOT execute (tool-error) -> bounded DEFER / fail-closed (D5); - 5. observability counters + one structured verdict log (D10). + 3. classify the outcome three ways (D3): ``suite-ran`` -> map exit-code -> + ``staging_status:``, write ``15-staging-log.md``, initiate the existing gate + via ``advance_stage`` (D7); + 4. ``permanent-env`` -> immediate infra-HOLD (no rollback, no developer-retry); + ``transient-infra`` -> bounded DEFER, then infra-HOLD on exhaustion (D4); + 5. observability counters + one structured verdict log (D8). Never raises into the caller (the launcher marks the job done/failed).""" started = time.time() _bump("runs") @@ -552,9 +708,10 @@ def run_staging_gate(job: dict) -> None: try: result = run_staging_suite() duration_s = round(time.time() - started, 1) - suite_ran = (result.returncode is not None) and (not result.timed_out) + # ORCH-123 (D3): three-way classification (env != transient infra != code-fail). + outcome = classify_staging_outcome(result, _channel_ssh_configured()) - if suite_ran: + if outcome == "suite-ran": # 3. trust the exit-code (ORCH-061 already inside staging_check.py). status = map_exit_code_to_status(result.returncode) _bump("success" if status == "SUCCESS" else "failed") @@ -568,14 +725,27 @@ def run_staging_gate(job: dict) -> None: _advance(task_id, repo, work_item_id, branch) return - # 4. tool-error (suite did not execute) -> DEFER / fail-closed (D5). + # The suite did NOT execute. Count it as a tool-error, then route by class + # (D4): permanent-env -> immediate infra-HOLD; transient-infra -> bounded DEFER. + # NEITHER rolls back to development or burns a developer-retry (BR-2 invariant). _bump("tool_error") + if outcome == "permanent-env": + _bump("permanent_env") + logger.warning( + "staging_runner verdict: work_item=%s repo=%s exit_code=%s " + "duration_s=%s outcome=permanent-env (ssh_configured=%s)", + work_item_id, repo, result.returncode, duration_s, _channel_ssh_configured(), + ) + _handle_permanent_env(task_id, repo, work_item_id, branch, result) + return + + # transient-infra (timeout / ssh transport / unknown) -> bounded DEFER. logger.warning( - "staging_runner verdict: work_item=%s repo=%s exit_code=%s status=%s " - "duration_s=%s outcome=tool-error (timed_out=%s)", - work_item_id, repo, result.returncode, "TOOL-ERROR", duration_s, result.timed_out, + "staging_runner verdict: work_item=%s repo=%s exit_code=%s " + "duration_s=%s outcome=transient-infra (timed_out=%s)", + work_item_id, repo, result.returncode, duration_s, result.timed_out, ) - _handle_tool_error(task_id, repo, work_item_id, branch, result) + _handle_transient_infra(task_id, repo, work_item_id, branch, result) except Exception as e: # noqa: BLE001 - never-raise into the worker (AC-7) logger.error( "staging_runner.run_staging_gate: unexpected error for task %s (%s): %s", @@ -584,24 +754,107 @@ def run_staging_gate(job: dict) -> None: # --------------------------------------------------------------------------- -# Observability (D10 / FR-7 / AC-10) +# Prod-like preflight of the host-side execution channel (D5 / FR-4 / AC-5) +# --------------------------------------------------------------------------- +def _alert_preflight(reason: str) -> None: + """Best-effort Telegram alert that the host-side staging channel is unworkable at + startup (so a real task never silently false-routes later). never-raise.""" + try: + from .notifications import send_telegram + send_telegram( + f"⚠️ staging-runner preflight: {reason}. Хост-сторона исполнения " + f"staging-сюиты неработоспособна — задачи на deploy-staging будут удержаны " + f"(инфра, НЕ дефект кода). Проверьте ssh/docker/staging-стенд." + ) + except Exception as e: # noqa: BLE001 - never-raise + logger.warning("staging_runner: preflight alert failed: %s", e) + + +def preflight() -> tuple[bool, str]: + """Prod-like preflight of the host-side staging execution channel (D5 / AC-5). + + Probes the channel with a short bounded ssh probe (``command -v docker`` + + ``docker inspect -f '{{.State.Running}}' ``) so a broken + environment (no docker on the host / staging container down / ssh unreachable / no + ssh target configured) surfaces at SERVICE START — NOT postfactum as a false + rollback of a real task. Records ``_PREFLIGHT_STATE`` + alerts on failure. + + Purely observational (FR-4): it NEVER gates/blocks the pipeline, touches stages / + QG, or raises — called best-effort from ``main.lifespan``. Self-hosting scope: + ``applies()`` first (a disabled runner / out-of-scope repo -> n/a). Returns + ``(ok, reason)`` (``ok=True`` for n/a / in-container mode / a healthy channel). + """ + try: + from .qg.checks import SELF_HOSTING_REPO + if not applies(SELF_HOSTING_REPO): + _PREFLIGHT_STATE.update(ok=None, reason="n/a (runner disabled / out of scope)") + return True, "n/a" + # In-container mode (host-side disabled): nothing to probe host-side. + if not bool(getattr(settings, "staging_runner_exec_host_side", True)): + _PREFLIGHT_STATE.update(ok=True, reason="in-container mode (host-side disabled)") + return True, "in-container mode" + target = _ssh_target() + if not target: + reason = "no ssh target configured (deploy_ssh_host empty) — host-side staging unworkable" + _PREFLIGHT_STATE.update(ok=False, reason=reason) + _alert_preflight(reason) + return False, reason + probe = ( + "command -v docker >/dev/null 2>&1 && " + f"docker inspect -f '{{{{.State.Running}}}}' {shlex.quote(STAGING_SERVICE)}" + ) + cmd = ["ssh", "-o", "StrictHostKeyChecking=no", target, probe] + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=_PREFLIGHT_TIMEOUT_S) + except subprocess.TimeoutExpired: + reason = "preflight ssh probe timed out" + _PREFLIGHT_STATE.update(ok=False, reason=reason) + _alert_preflight(reason) + return False, reason + except (subprocess.SubprocessError, OSError) as e: + reason = f"preflight ssh probe error: {e}" + _PREFLIGHT_STATE.update(ok=False, reason=reason) + _alert_preflight(reason) + return False, reason + out = (r.stdout or "").strip().lower() + if r.returncode == 0 and out == "true": + _PREFLIGHT_STATE.update(ok=True, reason="host-side channel ok (docker present, staging running)") + return True, "ok" + reason = f"host-side staging channel not ready (rc={r.returncode}, running={out!r})" + _PREFLIGHT_STATE.update(ok=False, reason=reason) + _alert_preflight(reason) + return False, reason + except Exception as e: # noqa: BLE001 - never-raise; preflight must never block start + logger.warning("staging_runner.preflight error: %s", e) + _PREFLIGHT_STATE.update(ok=None, reason=f"preflight error: {e}") + return True, "preflight skipped (error)" + + +# --------------------------------------------------------------------------- +# Observability (D8 / FR-7 / AC-10) # --------------------------------------------------------------------------- def snapshot() -> dict: """Read-only staging-runner summary for ``GET /queue`` (FR-7 / AC-10). - Additive block; existing ``/queue`` keys are untouched. never-raise: any error -> - a minimal dict with the kill-switch state.""" + Additive block; existing ``/queue`` keys are untouched. ORCH-123 adds + ``exec_host_side`` (the strategy flag), ``permanent_env`` (the infra-HOLD counter, + distinct from ``failed``=code-fail and ``deferred``=transient-infra) and the last + ``preflight`` verdict. never-raise: any error -> a minimal dict with the + kill-switch state.""" try: return { "enabled": bool(settings.staging_runner_enabled), "repos": getattr(settings, "staging_runner_repos", "") or "", "timeout_s": getattr(settings, "staging_runner_timeout_s", _DEFAULT_TIMEOUT_S), "infra_max_retries": getattr(settings, "staging_runner_infra_max_retries", 2), + "exec_host_side": bool(getattr(settings, "staging_runner_exec_host_side", True)), "runs": _STAGING_RUNNER_COUNTERS["runs"], "success": _STAGING_RUNNER_COUNTERS["success"], "failed": _STAGING_RUNNER_COUNTERS["failed"], "tool_error": _STAGING_RUNNER_COUNTERS["tool_error"], "deferred": _STAGING_RUNNER_COUNTERS["deferred"], + "permanent_env": _STAGING_RUNNER_COUNTERS["permanent_env"], + "preflight": dict(_PREFLIGHT_STATE), } except Exception as e: # noqa: BLE001 - never-raise -> minimal dict logger.warning("staging_runner.snapshot error: %s", e) diff --git a/tests/test_orch115_staging_runner.py b/tests/test_orch115_staging_runner.py index da8c415..719d21f 100644 --- a/tests/test_orch115_staging_runner.py +++ b/tests/test_orch115_staging_runner.py @@ -308,7 +308,11 @@ def test_tc10_timeout_defers_without_advance(monkeypatch): assert staging_runner._STAGING_RUNNER_COUNTERS["tool_error"] == 1 -def test_tc10_tool_error_budget_exhausted_fails_closed(monkeypatch, tmp_path): +def test_tc10_tool_error_budget_exhausted_infra_holds(monkeypatch, tmp_path): + # ORCH-123 (D4) SUPERSEDES ORCH-115 D5: a transient-infra DEFER budget that is + # exhausted no longer writes a fail-closed FAILED log + advances (a false rollback + # to development of an unresolved infra hiccup — BR-2). The task is now HELD on + # deploy-staging (infra-HOLD): NO advance, NO FAILED log, an operator alert instead. tid = _make_task("deploy-staging") monkeypatch.setattr(cfg.settings, "staging_runner_infra_max_retries", 0) # exhausted immediately monkeypatch.setattr(staging_runner, "run_staging_suite", @@ -318,8 +322,13 @@ def test_tc10_tool_error_budget_exhausted_fails_closed(monkeypatch, tmp_path): staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) - advance.assert_called_once() # fail-closed: write FAILED + advance (existing rollback) - assert "staging_status: FAILED" in _read_log(tmp_path, "orchestrator", "feature/ORCH-115-x", "ORCH-115") + advance.assert_not_called() # infra-HOLD: NO rollback to development + # No staging-log was written (a FAILED log would mislabel infra as a code-fail). + from src.git_worktree import get_worktree_path + log_path = os.path.join(get_worktree_path("orchestrator", "feature/ORCH-115-x"), + "docs/work-items/ORCH-115/15-staging-log.md") + assert not os.path.exists(log_path) + assert staging_runner._STAGING_RUNNER_COUNTERS["permanent_env"] == 1 def test_tc10_run_gate_never_raises(monkeypatch): @@ -369,6 +378,11 @@ def test_tc11_timeout_passed_to_proc_group(monkeypatch): return ProcResult(returncode=0, stdout="", stderr="", timed_out=False) monkeypatch.setattr(staging_runner.proc_group, "run_in_process_group", fake_run) + # ORCH-123 (D1): with no ssh target the command falls back to the in-container + # `docker exec` shape (the host-side ssh-wrap is covered in test_orch123_*). The + # invariant this test guards — timeout + tree_kill propagate to proc_group — is + # unchanged. + monkeypatch.setattr(cfg.settings, "deploy_ssh_host", "") staging_runner.run_staging_suite() assert captured["timeout"] == 321 assert captured["tree_kill"] is True @@ -420,13 +434,14 @@ def test_tc13_structured_verdict_log_distinguishes_outcomes(monkeypatch, caplog, assert any("outcome=code-pass" in r.message for r in caplog.records) caplog.clear() - # tool-error + # ORCH-123 (D8): a timeout is now classified `transient-infra` (the outcome + # taxonomy is code-pass/code-fail/transient-infra/permanent-env, not "tool-error"). monkeypatch.setattr(cfg.settings, "staging_runner_infra_max_retries", 2) monkeypatch.setattr(staging_runner, "run_staging_suite", lambda: ProcResult(returncode=None, stdout="", stderr="", timed_out=True)) with caplog.at_level("WARNING", logger="orchestrator.staging_runner"): staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) - assert any("outcome=tool-error" in r.message for r in caplog.records) + assert any("outcome=transient-infra" in r.message for r in caplog.records) def test_tc13_snapshot_never_raises(monkeypatch): diff --git a/tests/test_orch123_staging_runner_exec.py b/tests/test_orch123_staging_runner_exec.py new file mode 100644 index 0000000..9950184 --- /dev/null +++ b/tests/test_orch123_staging_runner_exec.py @@ -0,0 +1,503 @@ +"""ORCH-123 — staging-runner execution strategy must not depend on a docker CLI +inside the prod app-container (host-side ssh + three-way env classification). + +Background (incident ORCH-116): the ORCH-115 deterministic staging-runner ran the +staging suite via ``docker exec`` FROM INSIDE the prod ``orchestrator`` container, +which ships only ``openssh-client git curl`` — NOT a ``docker`` CLI (Dockerfile:11). +``Popen(["docker", ...])`` hit ``FileNotFoundError`` -> a PERMANENT environment defect +that ORCH-115 mis-routed as a code-fail rollback ``deploy-staging -> development`` +(burning developer-retries). The fix (ADR-001 / adr-0049): + + * D1 — execute the suite HOST-SIDE over the existing trusted ssh channel + (ORCH-036/058), wrapping the SAME ``docker exec ...`` command; + * D3 — three-way classification ``suite-ran`` / ``permanent-env`` / ``transient-infra``; + * D4 — environment/transient-infra NEVER ends in a code-fail rollback (infra-HOLD); + a really-executed failing suite STILL rolls back (anti-over-tolerance); + * D5 — a prod-like preflight of the host-side channel at startup. + +The pipeline gate / artifact contract / STAGE_TRANSITIONS / DB schema are byte-for-byte +unchanged (NFR-1). No live ssh / docker / network: the suite subprocess, advance_stage +and notifications are mocked; spawn-errors are reproduced with a non-existent binary. +""" +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_orch123_staging_runner_exec.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db # noqa: E402 +from src import staging_runner # noqa: E402 +from src import stage_engine # noqa: E402 +from src import config as cfg # noqa: E402 +from src.proc_group import ProcResult # noqa: E402 + + +# --------------------------------------------------------------------------- +# Fixtures (mirror tests/test_orch115_staging_runner.py) +# --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + monkeypatch.setattr("src.git_worktree.settings.worktrees_dir", str(tmp_path), raising=False) + # Runner ON, self-hosting scope, host-side strategy ON (defaults). + monkeypatch.setattr(cfg.settings, "staging_runner_enabled", True, raising=False) + monkeypatch.setattr(cfg.settings, "staging_runner_repos", "", raising=False) + monkeypatch.setattr(cfg.settings, "staging_runner_infra_max_retries", 2, raising=False) + monkeypatch.setattr(cfg.settings, "staging_runner_exec_host_side", True, raising=False) + # A configured ssh target (prod compose sets ORCH_DEPLOY_SSH_HOST=127.0.0.1). + monkeypatch.setattr(cfg.settings, "deploy_ssh_host", "127.0.0.1", raising=False) + monkeypatch.setattr(cfg.settings, "deploy_ssh_user", "slin", raising=False) + for k in staging_runner._STAGING_RUNNER_COUNTERS: + staging_runner._STAGING_RUNNER_COUNTERS[k] = 0 + staging_runner._PREFLIGHT_STATE.update(ok=None, reason="not-probed") + yield + + +def _make_task(stage, repo="orchestrator", wi="ORCH-123", branch="feature/ORCH-123-x"): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + tid = cur.lastrowid + conn.commit() + conn.close() + return tid + + +def _job_dict(job_id, agent, repo, task_id): + return {"id": job_id, "agent": agent, "repo": repo, "task_id": task_id, "task_content": "x"} + + +def _infra_jobs(task_id): + conn = get_db() + n = conn.execute( + "SELECT COUNT(*) FROM jobs WHERE task_id=? AND task_content LIKE ?", + (task_id, f"%{staging_runner._INFRA_RETRY_MARKER}%"), + ).fetchone()[0] + conn.close() + return n + + +def _read_log(repo, branch, wi): + from src.git_worktree import get_worktree_path + p = os.path.join(get_worktree_path(repo, branch), f"docs/work-items/{wi}/15-staging-log.md") + with open(p, "r", encoding="utf-8") as f: + return f.read() + + +def _log_exists(repo, branch, wi): + from src.git_worktree import get_worktree_path + return os.path.exists( + os.path.join(get_worktree_path(repo, branch), f"docs/work-items/{wi}/15-staging-log.md") + ) + + +# --------------------------------------------------------------------------- +# TC-01 — MANDATORY regression (red->green): no docker CLI in the container +# --------------------------------------------------------------------------- +def test_tc01_regression_no_docker_cli_in_container(monkeypatch): + """Reproduces incident ORCH-116. PRE-FIX: an in-container ``docker exec`` where the + container has no docker CLI -> proc_group spawn-error -> tool-error -> infra-DEFER x2 + -> false FAILED rollback to development. POST-FIX: classified ``permanent-env`` -> + infra-HOLD: NO advance, NO re-queue, NO developer-retry. (Red on pre-fix code — + there is no permanent-env class / counter there and the path advances.)""" + tid = _make_task("deploy-staging") + # Force the pre-fix in-container shape, then make the spawn fail exactly as if + # `docker` were absent — no network / ssh / docker needed. + monkeypatch.setattr(cfg.settings, "staging_runner_exec_host_side", False) + monkeypatch.setattr(staging_runner, "build_staging_command", + lambda: ["orch123-no-such-binary", "exec", "orchestrator-staging"]) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + advance.assert_not_called() # NO false rollback to development + assert _infra_jobs(tid) == 0 # DEFER cycle skipped (FR-3) + assert not _log_exists("orchestrator", "feature/ORCH-123-x", "ORCH-123") # no FAILED log + assert staging_runner._STAGING_RUNNER_COUNTERS["permanent_env"] == 1 + assert staging_runner._STAGING_RUNNER_COUNTERS["tool_error"] == 1 + assert stage_engine.developer_retry_count(tid) == 0 # developer-retry not burned + + +# --------------------------------------------------------------------------- +# TC-02 — strategy does not require a docker CLI on the container PATH +# --------------------------------------------------------------------------- +def test_tc02_host_side_command_does_not_assume_in_container_docker(): + # With host-side ON + an ssh target, the suite is dispatched via ssh — the prod + # container's own PATH is never asked for a `docker` binary. + cmd = staging_runner.build_staging_command() + assert cmd[0] == "ssh" # dispatched over the trusted channel + assert "slin@127.0.0.1" in cmd + # The INNER command is the SAME staging suite (contract unchanged). + remote = cmd[-1] + assert "docker exec orchestrator-staging" in remote + assert "staging_check.py" in remote + assert "--mode stub" in remote + # The local executable invoked is `ssh` (in the container image), NOT `docker`. + assert cmd[0] != "docker" + + +# --------------------------------------------------------------------------- +# TC-03 — environment/tool-error does NOT cause a misleading code-fail rollback +# --------------------------------------------------------------------------- +def test_tc03_env_defect_no_code_fail_rollback_and_alerts(monkeypatch): + tid = _make_task("deploy-staging") + monkeypatch.setattr( + staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=None, stdout="", + stderr="[Errno 2] No such file or directory: 'docker'", timed_out=False), + ) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + sent = [] + monkeypatch.setattr("src.notifications.send_telegram", lambda msg, **kw: sent.append(msg)) + + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + advance.assert_not_called() # not a code-fail -> no rollback + assert _infra_jobs(tid) == 0 # permanent defect -> no pointless retry + assert stage_engine.developer_retry_count(tid) == 0 + # A distinguishable infra/environment alert ("NOT a code defect") was sent. + assert sent and any("НЕ дефект кода" in m for m in sent) + + +# --------------------------------------------------------------------------- +# TC-04 — anti-over-tolerance: a really-executed failing suite STILL rolls back +# --------------------------------------------------------------------------- +def test_tc04_real_suite_failure_still_rolls_back(monkeypatch): + tid = _make_task("deploy-staging") + # The suite executed (a real exit-code) and failed; clean stderr (no env-marker). + monkeypatch.setattr( + staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=1, stdout="B7 FAIL: assertion", stderr="", timed_out=False), + ) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + advance.assert_called_once() # same rollback path as ORCH-115 + kwargs = advance.call_args.kwargs + assert kwargs["current_stage"] == "deploy-staging" + assert kwargs["finished_agent"] == "deployer" + assert "staging_status: FAILED" in _read_log("orchestrator", "feature/ORCH-123-x", "ORCH-123") + assert staging_runner._STAGING_RUNNER_COUNTERS["failed"] == 1 + assert staging_runner._STAGING_RUNNER_COUNTERS["permanent_env"] == 0 + + +def test_tc04_exit1_with_env_marker_is_not_a_code_fail(monkeypatch): + # The exit=1 collision: `docker exec` "No such container"=1 must NOT be read as a + # suite fail=1. The stderr env-marker disambiguates it -> permanent-env (no rollback). + tid = _make_task("deploy-staging") + monkeypatch.setattr( + staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=1, stdout="", + stderr="Error: No such container: orchestrator-staging", timed_out=False), + ) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + monkeypatch.setattr("src.notifications.send_telegram", lambda *a, **kw: None) + + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + advance.assert_not_called() + assert staging_runner._STAGING_RUNNER_COUNTERS["permanent_env"] == 1 + + +# --------------------------------------------------------------------------- +# TC-05 — happy-path: suite ran, exit 0 -> SUCCESS -> advance +# --------------------------------------------------------------------------- +def test_tc05_happy_path_success_advances(monkeypatch): + tid = _make_task("deploy-staging") + monkeypatch.setattr( + staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=0, stdout="ALL OK", stderr="", timed_out=False), + ) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + advance.assert_called_once() + assert advance.call_args.kwargs["finished_agent"] == "deployer" + assert "staging_status: SUCCESS" in _read_log("orchestrator", "feature/ORCH-123-x", "ORCH-123") + assert staging_runner._STAGING_RUNNER_COUNTERS["success"] == 1 + + +# --------------------------------------------------------------------------- +# TC-06 — three-way classification (D3) +# --------------------------------------------------------------------------- +def _pr(rc=None, stderr="", timed_out=False): + return ProcResult(returncode=rc, stdout="", stderr=stderr, timed_out=timed_out) + + +def test_tc06_classification_three_way(): + C = staging_runner.classify_staging_outcome + # suite-ran: any recognised int (except 255) with no env-marker -> trust it. + assert C(_pr(rc=0), True) == "suite-ran" + assert C(_pr(rc=1), True) == "suite-ran" # anti-over-tolerance (BR-3) + assert C(_pr(rc=2, stderr="B7 FAIL"), True) == "suite-ran" + # permanent-env: spawn-error (rc None, not a timeout). + assert C(_pr(rc=None), True) == "permanent-env" + # permanent-env: shell command-not-found / cannot-execute codes. + assert C(_pr(rc=127), True) == "permanent-env" + assert C(_pr(rc=126), True) == "permanent-env" + # permanent-env: env-marker regardless of the exit-code. + assert C(_pr(rc=1, stderr="Cannot connect to the Docker daemon"), True) == "permanent-env" + assert C(_pr(rc=1, stderr="docker: command not found"), True) == "permanent-env" + # permanent-env: host-side ssh target not configured (R-6). + assert C(_pr(rc=None), False) == "permanent-env" + # transient-infra: timeout / ssh transport (255) -> retry is meaningful. + assert C(_pr(timed_out=True), True) == "transient-infra" + assert C(_pr(rc=255), True) == "transient-infra" + + +def test_tc06_classification_never_raises(): + class Boom: + @property + def returncode(self): + raise RuntimeError("boom") + # An exploding result -> fail-safe transient-infra (DEFER, never a silent suite-ran). + assert staging_runner.classify_staging_outcome(Boom(), True) == "transient-infra" + + +def test_tc06_transient_infra_defers_not_rolls_back(monkeypatch): + tid = _make_task("deploy-staging") + monkeypatch.setattr(staging_runner, "run_staging_suite", + lambda: _pr(rc=255, stderr="ssh: connect: Connection refused")) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + advance.assert_not_called() # transient -> DEFER, never rollback + assert _infra_jobs(tid) == 1 # a fresh deployer job re-queued + assert staging_runner._STAGING_RUNNER_COUNTERS["deferred"] == 1 + + +# --------------------------------------------------------------------------- +# TC-07 — prod-like preflight (D5 / AC-5) +# --------------------------------------------------------------------------- +def test_tc07_preflight_no_ssh_target_signals_before_a_real_task(monkeypatch): + monkeypatch.setattr(cfg.settings, "deploy_ssh_host", "") # host-side, but no target + sent = [] + monkeypatch.setattr("src.notifications.send_telegram", lambda msg, **kw: sent.append(msg)) + + ok, reason = staging_runner.preflight() + + assert ok is False + assert "no ssh target" in reason + assert staging_runner._PREFLIGHT_STATE["ok"] is False + assert sent # operator alerted at startup + + +def test_tc07_preflight_probe_success(monkeypatch): + class _R: + returncode = 0 + stdout = "true\n" + stderr = "" + monkeypatch.setattr(staging_runner.subprocess, "run", lambda *a, **kw: _R()) + ok, reason = staging_runner.preflight() + assert ok is True + assert staging_runner._PREFLIGHT_STATE["ok"] is True + + +def test_tc07_preflight_probe_failure_alerts(monkeypatch): + class _R: + returncode = 1 + stdout = "" # docker missing on host / container down + stderr = "command not found" + monkeypatch.setattr(staging_runner.subprocess, "run", lambda *a, **kw: _R()) + sent = [] + monkeypatch.setattr("src.notifications.send_telegram", lambda msg, **kw: sent.append(msg)) + ok, _reason = staging_runner.preflight() + assert ok is False + assert sent + + +def test_tc07_preflight_noop_when_disabled(monkeypatch): + monkeypatch.setattr(cfg.settings, "staging_runner_enabled", False) + ok, reason = staging_runner.preflight() # out of scope -> n/a, never blocks + assert ok is True + assert reason == "n/a" + + +def test_tc07_preflight_in_container_mode_noop(monkeypatch): + monkeypatch.setattr(cfg.settings, "staging_runner_exec_host_side", False) + ok, reason = staging_runner.preflight() + assert ok is True + assert "in-container" in reason + + +# --------------------------------------------------------------------------- +# TC-08 — self-hosting safety: no dangerous literals in the host-side command +# --------------------------------------------------------------------------- +def test_tc08_host_side_command_has_no_dangerous_literals(): + cmd = " ".join(staging_runner.build_staging_command()) + forbidden = ("compose", "up -d", "--build", "8500", "force", "push", ".env", + "rm ", "restart") + for token in forbidden: + assert token not in cmd, f"forbidden literal {token!r} in host-side command: {cmd}" + assert "ssh" in cmd and "docker exec" in cmd and "staging_check.py" in cmd + assert "8501" in cmd and "--mode stub" in cmd + + +# --------------------------------------------------------------------------- +# TC-09 — kill-switch / scope / strategy flag default (AC-10) +# --------------------------------------------------------------------------- +def test_tc09_killswitch_scope_and_flag_default(monkeypatch): + # kill-switch off -> should_intercept False -> the prior LLM deployer via _spawn. + monkeypatch.setattr(cfg.settings, "staging_runner_enabled", False) + tid = _make_task("deploy-staging") + assert staging_runner.should_intercept(_job_dict(1, "deployer", "orchestrator", tid)) is False + # enabled, empty CSV -> self-hosting only. + monkeypatch.setattr(cfg.settings, "staging_runner_enabled", True) + assert staging_runner.applies("orchestrator") is True + assert staging_runner.applies("enduro-trails") is False + # the strategy flag defaults to host-side (prod). + assert cfg.settings.staging_runner_exec_host_side is True + + +def test_tc09_host_side_off_falls_back_to_in_container(monkeypatch): + monkeypatch.setattr(cfg.settings, "staging_runner_exec_host_side", False) + cmd = staging_runner.build_staging_command() + assert cmd[:2] == ["docker", "exec"] # rollback knob -> prior 1:1 shape + + +def test_tc09_host_side_on_but_no_target_falls_back(monkeypatch): + monkeypatch.setattr(cfg.settings, "deploy_ssh_host", "") + cmd = staging_runner.build_staging_command() + assert cmd[:2] == ["docker", "exec"] # no target -> in-container fallback + + +# --------------------------------------------------------------------------- +# TC-10 — artifact contract: staging_status frontmatter + 52c schema (AC-7) +# --------------------------------------------------------------------------- +def test_tc10_artifact_contract_unchanged(): + body = staging_runner.build_staging_log("ORCH-123", 0, "SUCCESS", "ok") + assert "staging_status: SUCCESS" in body # UPPERCASE machine key, unchanged + for field in ("work_item: ORCH-123", "stage: deploy-staging", + "author_agent: staging-runner", "model_used: n/a"): + assert field in body + # The UNCHANGED gate parser reads the runner's frontmatter. + from src.qg.checks import _parse_staging_status + ok, _r = _parse_staging_status(body) + assert ok is True + bad, _r2 = _parse_staging_status(staging_runner.build_staging_log("ORCH-123", 1, "FAILED")) + assert bad is False + + +# --------------------------------------------------------------------------- +# TC-11 — pipeline anti-regress: gate / transitions / schema untouched (AC-7) +# --------------------------------------------------------------------------- +def test_tc11_pipeline_contract_unchanged(): + from src.stages import STAGE_TRANSITIONS + from src.qg.checks import QG_CHECKS + assert STAGE_TRANSITIONS["deploy-staging"] == { + "next": "deploy", "agent": "deployer", "qg": "check_staging_status" + } + assert "check_staging_status" in QG_CHECKS + conn = get_db() + tables = {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'").fetchall()} + conn.close() + assert not any("staging_runner" in t for t in tables) # no DB migration + + +# --------------------------------------------------------------------------- +# TC-12 — never-raise / queue not wedged + observability distinguishes classes +# --------------------------------------------------------------------------- +def test_tc12_run_gate_never_raises(monkeypatch): + tid = _make_task("deploy-staging") + + def boom(): + raise RuntimeError("ssh exploded") + monkeypatch.setattr(staging_runner, "run_staging_suite", boom) + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) # must NOT raise + + +def test_tc12_snapshot_distinguishes_classes(): + snap = staging_runner.snapshot() + for k in ("enabled", "exec_host_side", "failed", "deferred", "permanent_env", "preflight"): + assert k in snap, f"snapshot missing key {k}" + # The three non-success classes are distinct keys (code-fail vs transient vs env). + assert {"failed", "deferred", "permanent_env"} <= set(snap) + + +def test_tc12_snapshot_never_raises(monkeypatch): + class Boom: + def __getattr__(self, name): + raise RuntimeError("boom") + monkeypatch.setattr(staging_runner, "settings", Boom()) + assert staging_runner.snapshot()["enabled"] is False + + +# --------------------------------------------------------------------------- +# R-2 — a held deploy-staging task is NOT rolled back to development by the +# reconciler (the defect would re-appear if it were). AC-3 / AC-11. +# --------------------------------------------------------------------------- +def test_r2_held_deploy_staging_not_rolled_back(monkeypatch): + tid = _make_task("deploy-staging") + # No 15-staging-log.md was written (infra-HOLD) -> check_staging_status is red. + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + + result = stage_engine.advance_if_gate_passed( + tid, "deploy-staging", "orchestrator", "ORCH-123", "feature/ORCH-123-x" + ) + + assert result is None # red gate -> stay, no advance call + advance.assert_not_called() # NEVER rolled back to development + conn = get_db() + stage = conn.execute("SELECT stage FROM tasks WHERE id=?", (tid,)).fetchone()[0] + conn.close() + assert stage == "deploy-staging" # held in place + + +# --------------------------------------------------------------------------- +# TC-13 — documentation of the execution boundary (AC-12) +# --------------------------------------------------------------------------- +def _repo_root(): + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def test_tc13_docs_describe_execution_boundary(): + root = _repo_root() + with open(os.path.join(root, "docs/operations/INFRA.md"), encoding="utf-8") as f: + infra = f.read().lower() + with open(os.path.join(root, "docs/architecture/README.md"), encoding="utf-8") as f: + readme = f.read().lower() + # INFRA.md fixes the boundary: no docker CLI in the container; docker ops host-side. + assert "host-side" in infra + assert "docker" in infra and "ssh" in infra + assert "orch-123" in infra + # README describes the host-side staging-runner strategy. + assert "orch-123" in readme + assert "host-side" in readme + + +# --------------------------------------------------------------------------- +# TC-14 — anti-drift ORCH-115: invariants intact (kill-switch -> LLM, shared map) +# --------------------------------------------------------------------------- +def test_tc14_orch115_invariants_intact(monkeypatch): + # Kill-switch off -> the single LLM transport path is restored (no intercept). + monkeypatch.setattr(cfg.settings, "staging_runner_enabled", False) + tid = _make_task("deploy-staging") + assert staging_runner.should_intercept(_job_dict(1, "deployer", "orchestrator", tid)) is False + # The exit-code mapping is still the SAME shared self_deploy contract (no drift). + from src import self_deploy + for v in (0, 1, None, "x"): + assert staging_runner.map_exit_code_to_status(v) == self_deploy.map_exit_code_to_status(v)