diff --git a/.env.example b/.env.example index 4bd4d96..b27f915 100644 --- a/.env.example +++ b/.env.example @@ -557,6 +557,27 @@ ORCH_COVERAGE_EPSILON=0.5 ORCH_COVERAGE_TOOL_FAIL_CLOSED=false ORCH_COVERAGE_RUN_TIMEOUT_S=900 +# ORCH-115: deterministic staging-runner replacing the LLM `deployer` on the +# `deploy-staging` stage (self-hosting orchestrator). Intercepted in launch_job +# BEFORE _spawn (deploy-finalizer / post-deploy-monitor precedent): runs the same +# staging suite, maps exit-code -> staging_status:, writes 15-staging-log.md and +# initiates the UNCHANGED check_staging_status gate. Replaces only the producer of +# the artifact; the gate / STAGE_TRANSITIONS / DB schema are byte-for-byte unchanged. +# See ADR-001-deterministic-staging-runner.md / adr-0048. +# STAGING_RUNNER_ENABLED -> kill-switch; false -> the prior LLM deployer +# runs on deploy-staging via _spawn 1:1. +# STAGING_RUNNER_REPOS -> CSV scope; empty -> self-hosting only. +# STAGING_RUNNER_TIMEOUT_S -> wall-clock budget for the docker-exec suite +# (malformed/non-positive -> default 600 + WARNING). +# STAGING_RUNNER_INFRA_MAX_RETRIES -> tool-error (suite did NOT execute) bounded DEFER +# budget before a fail-closed FAILED (anti ORCH-110). +# STAGING_RUNNER_INFRA_RETRY_DELAY_S-> delay before the re-queued deployer job. +ORCH_STAGING_RUNNER_ENABLED=true +ORCH_STAGING_RUNNER_REPOS= +ORCH_STAGING_RUNNER_TIMEOUT_S=600 +ORCH_STAGING_RUNNER_INFRA_MAX_RETRIES=2 +ORCH_STAGING_RUNNER_INFRA_RETRY_DELAY_S=30 + # ORCH-057 (follow-up ORCH-040): legacy root-owned ownership detect + actionable # worktree error. After the uid migration (user: "1000:1000") legacy root:root files # in /repos broke worktree creation under uid 1000 with a raw "Permission denied". diff --git a/.openclaw/agents/deployer.md b/.openclaw/agents/deployer.md index e456d37..992afee 100644 --- a/.openclaw/agents/deployer.md +++ b/.openclaw/agents/deployer.md @@ -45,6 +45,16 @@ then emit `staging_status:` / `deploy_status:`. Run the staging test suite against the live staging environment and write the verdict. +> **ORCH-115 — deterministic runner leads this stage for in-scope repos.** On `deploy-staging` for +> the self-hosting `orchestrator` repo, this stage is now driven by **deterministic code** +> (`src/staging_runner.py`, intercepted in `launch_job` BEFORE `_spawn`, mirroring the prod Phase +> A/B/C pattern) — it runs the SAME canonical staging suite below, maps the exit code to +> `staging_status:` via the same `0 → SUCCESS / non-zero → FAILED` contract, writes +> `15-staging-log.md`, and initiates the unchanged `check_staging_status` gate. The LLM steps below +> remain the **fallback** under a disabled kill-switch (`ORCH_STAGING_RUNNER_ENABLED=false`) or for +> non-self repos. The artifact contract / gate / machine key `staging_status:` are unchanged. Details: +> `docs/work-items/ORCH-115/06-adr/ADR-001-deterministic-staging-runner.md`. + **Steps:** 1. Run the staging suite. **CANONICAL: run INSIDE the `orchestrator-staging` container via diff --git a/.task-dev.md b/.task-dev.md index c218318..75f0fd3 100644 --- a/.task-dev.md +++ b/.task-dev.md @@ -1,4 +1,4 @@ -Work item: ORCH-112 +Work item: ORCH-115 Repo: orchestrator -Branch: feature/ORCH-112-bug-failed-cancelled-task-arti +Branch: feature/ORCH-115-orch-replace-llm-deployer-with Stage: development \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e6b52b..2902751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Формат: [Keep a Changelog](https://keepachangelog.com/). Записи — на смысловой PR/задачу. ## [Unreleased] +- **Детерминированный staging-раннер вместо LLM-деплойера на `deploy-staging`** (ORCH-115, `feat`): первый реализованный срез determinization-roadmap (ORCH-118 A6, `replace-deterministic-now`) — на стадии `deploy-staging` для self-hosting `orchestrator` **LLM-агент `deployer` заменён детерминированным кодом** (`src/staging_runner.py`). Работа агента на этой стадии была чисто механической (запуск staging-сюиты + маппинг exit-кода `staging_check.py` → `staging_status:`); каждый прогон жёг токены/время opus-агента (~40–120k / 3–15 мин) и встраивал недетерминизм LLM в точку ветвления гейта. **Инвариант (NFR-1):** это замена *продюсера* артефакта, **не** гейта — контракт `15-staging-log.md`, гейт `check_staging_status`/`_parse_staging_status`, `STAGE_TRANSITIONS`, machine-verdict `staging_status:`, схема БД — **байт-в-байт не тронуты**. Аддитивно, под kill-switch, never-raise, fail-closed, скоуп self-hosting. ADR: `docs/work-items/ORCH-115/06-adr/ADR-001-deterministic-staging-runner.md`, сквозной `docs/architecture/adr/adr-0048-deterministic-staging-runner.md`. + - **Перехват в `launch_job` до `_spawn` (D1):** `if job.agent=="deployer" and staging_runner.should_intercept(job)` → `_run_staging_runner_job` (зеркало `_run_deploy_finalizer_job`, прецедент `deploy-finalizer`/`post-deploy-monitor` `launcher.py:389/394`): синхронно ведёт `jobs`-строку через `mark_job`, возвращает `None` (нет `agent_runs`-строки, нет токенов). Дискриминатор «staging vs prod» — **стадия задачи** `deploy-staging` (роль `deployer` общая для `deploy-staging`/`deploy`), не имя роли; `should_intercept` never-raise → `False` → штатный `_spawn` (fail-safe к LLM-пути). Для не-self репо `applies==False` → прод-`deployer` никогда не перехватывается. + - **Leaf `src/staging_runner.py` (новый, чистый never-raise):** по образцу `self_deploy`/`proc_group`/`staging_verdict` (на импорте только `config`/`proc_group`; `db`/`git_worktree`/`stage_engine`/`qg.checks`/`notifications` — лениво). Исполняет ту же сюиту (`docker exec orchestrator-staging python3 //scripts/staging_check.py --base-url http://localhost: --mode stub`, арги из config — ORCH-101, без host-хардкодов) через `proc_group.run_in_process_group` (tree-kill, таймаут `staging_runner_timeout_s=600`, малформ/непозитив → дефолт + WARNING); маппит exit-код **единым** контрактом `self_deploy.map_exit_code_to_status` (`0→SUCCESS`/иначе/None→`FAILED`, fail-closed; ORCH-061 infra-tolerance остаётся внутри `staging_check.py`, раннер не пересуживает — BR-4); пишет `15-staging-log.md` (тот же machine-key `staging_status:` UPPERCASE + 52c-схема, `author_agent: staging-runner`/`model_used: n/a`) + best-effort commit/push в **фичеветку** (не в `main` — гейт читает worktree первым, усиливает AC-8/BR-7); вызывает **существующий** `advance_stage(current_stage="deploy-staging", finished_agent="deployer")` — без новых рёбер/исходов (transition-lease ORCH-114 берётся внутри `advance_stage`, раннер не трогает — граница O1). + - **Двухуровневый исход (D5, анти-ORCH-110):** сюита **исполнилась** (реальный exit-код) → verdict→advance (FAILED → тот же откат `deploy-staging → development` + developer-retry, что у FAILED-вердикта LLM, `stage_engine.py:932`); сюита **не исполнилась** (tool-error: spawn-error/таймаут/`returncode None`) → инфра-сбой ≠ код-фейл → bounded **DEFER** (re-queue `deployer`-джоба с задержкой + restart-safe маркер `staging-runner infra-retry` в `task_content`, счётчик подсчётом маркера — без правки схемы БД), на исчерпании `staging_runner_infra_max_retries=2` → fail-closed `FAILED` + advance + инфра-alert. Раннер **никогда** не делает тихий advance/ложный green, **никогда** не клинит очередь и **не** жжёт developer-retry на транзиентной инфре. + - **Self-hosting safety (BR-7/AC-8):** в командной строке раннера нет рестарта 8500 / `docker compose up orchestrator` / `--build` / force-push в `main` / правок `.env` — раннер только читает/исполняет staging-сюиту (8501) и пишет лог. Наблюдаемость (D10) — in-process счётчики (`runs`/`success`/`failed`/`tool_error`/`deferred`) + read-only блок `staging_runner` в `GET /queue` + один структурный лог-вердикт на прогон (различает код-фейл и tool-error). Флаги (`config.py`, дефолт = боевое): `staging_runner_enabled` (env `ORCH_STAGING_RUNNER_ENABLED`), `staging_runner_repos` (CSV; пусто → self-hosting only), `staging_runner_timeout_s`, `staging_runner_infra_max_retries`, `staging_runner_infra_retry_delay_s`. Откат = `ORCH_STAGING_RUNNER_ENABLED=false` → на `deploy-staging` снова LLM-`deployer` через `_spawn` **байт-в-байт**. + - **Норматив сопровождения ORCH-118 (NFR-6):** обновлены `docs/architecture/llm-call-sites.md` (A6 — реализован; машинный `ORCH-118-INVENTORY-BLOCK` сохраняет deployer как `avoidable=yes`/`axis=C` — LLM-ветвь жива как fallback), `llm-determinization-roadmap.md` (rank 1 deployer — ✅ реализован; машинный блок/инвариант «ровно один `first_slice = yes`» целы), `llm-usage-policy.md` (§5 — единственный транспорт S0 не нарушен, раннер LLM не зовёт), `.openclaw/agents/deployer.md` (LLM-ветвь `deploy-staging` — fallback), витрина `docs/overview/tech-pipeline.md`/`tech-agents.md`. Покрытие — `tests/test_orch115_staging_runner.py` (TC-01…TC-13) + зелёные `tests/test_llm_call_site_inventory.py`/`test_llm_determinization_docs.py` (TC-14). - **Карта LLM-консультаций + control-path-ось «avoidable» + roadmap + нормативная политика** (ORCH-118, `docs`+`test`, inventory-first, docs+tests only): зонтичный follow-up RCA-трека ORCH-114/117 — у оркестратора не было ни нормативного критерия «где LLM нужен, а где это avoidable control path», ни карты мест вызова LLM, прибитой к коду. Выпущена **доказательная карта** каждого места, где control-path потребляет (или способен потребить) суждение LLM, с control-path-разметкой и классификацией; **упорядоченный roadmap** детерминированных замен; **нормативная политика** использования LLM; набор **структурных анти-дрейф тестов**. Это **docs + tests only**: `src/**`-рантайм не меняется → `STAGE_TRANSITIONS` / реестр и имена `QG_CHECKS`/`check_*` / machine-verdict-ключи / схема БД — **байт-в-байт не тронуты**; раннеры замен **не** реализуются (FR-7); конкретные follow-up Plane-ID **не** фиксируются (R3/NFR-6 — кандидаты по роли). kill-switch не нужен (нет рантайм-поведения), как ORCH-077/079/101/102/103/011. ADR: `docs/work-items/ORCH-118/06-adr/ADR-001-llm-call-site-map-and-determinization-roadmap.md`, сквозной `docs/architecture/adr/adr-0047-llm-usage-policy-and-call-site-map.md`. - **Единица инвентаря — LLM-консультация** (control-path потребляет суждение LLM), а **не** «спавн процесса / существование Claude CLI» (R4, capability ≠ consultation). Карта разводит **три ортогональных факта**: (1) consultation ≠ transport/slot (единственный транспорт LLM-консультации в `src/**` — `launcher._spawn`, `launcher.py:472`/CLI-сборка `610-614`; иного транспорта нет; job-роли `deploy-finalizer`/`post-deploy-monitor` занимают слот, но перехватываются в `launch_job` **до** `_spawn`, `launcher.py:389/394` — консультации нет); (2) **control-path (C) ≠ artifact-producer (P)** по коду-потребителю в `src/qg/checks.py` (C: гейт ветвится на LLM-вердикте; P: детерминированный гейт судит артефакт независимо — файлы/CI); (3) деривируемость вердикта из tool-сигналов. - **Нормативное определение** «avoidable LLM control path» = двухбитный предикат (C-консультация **И** вердикт деривируем из exit-кодов `pytest`/smoke/`staging_check.py`/деплоя). Поимённый целевой набор (доказательно, прибит тестами): **avoidable = `{tester, deployer}`**; control-path-но-keep = `{reviewer}` (вердикт «приемлемость кода/решения» НЕ деривируем); не-control-path (P, keep) = `{analyst, architect, developer}`; уже детерминированы (эталон) = `{deploy-finalizer, post-deploy-monitor}`. diff --git a/CLAUDE.md b/CLAUDE.md index f23bbc4..2d5ac85 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -411,6 +411,52 @@ Plane, **не** Quality Gate и **не** стадия). `docs/work-items/ORCH-117/06-adr/ADR-001-sandbox-only-plane-write-guard.md`, сквозной `docs/architecture/adr/adr-0046-sandbox-only-plane-write-guard.md`. +## Детерминированный staging-раннер вместо LLM-деплойера (ORCH-115) +Первый реализованный срез determinization-roadmap (ORCH-118 A6, `replace-deterministic-now`): на +стадии `deploy-staging` для self-hosting `orchestrator` **LLM-агент `deployer` заменён +детерминированным кодом** (`src/staging_runner.py`). Работа агента на этой стадии была чисто +механической (запуск staging-сюиты + маппинг exit-кода) — теперь её делает leaf, перехватываемый в +`launch_job` **до `_spawn`** (прецедент `deploy-finalizer`/`post-deploy-monitor`, +`launcher.py:389/394`). **Инвариант (NFR-1):** замена *продюсера* артефакта, **не** гейта — +контракт `15-staging-log.md`, гейт/`_parse_staging_status`/имя `check_staging_status`, +`STAGE_TRANSITIONS`, machine-verdict `staging_status:`, схема БД — **байт-в-байт не тронуты**. +Аддитивно, под kill-switch, never-raise, fail-closed. +- **Перехват (D1):** `launch_job` — `if job.agent=="deployer" and staging_runner.should_intercept(job)` + → `_run_staging_runner_job` (зеркало `_run_deploy_finalizer_job`): синхронно ведёт `jobs`-строку + через `mark_job`, возвращает `None` (нет `agent_runs`). Дискриминатор «staging vs prod» — + **стадия задачи** `deploy-staging` (роль `deployer` общая для `deploy-staging`/`deploy`), не имя + роли; `should_intercept` never-raise → `False` → штатный `_spawn` (fail-safe к LLM-пути). +- **Раннер (D2-D7):** leaf по образцу `self_deploy`/`proc_group`/`staging_verdict` (на импорте только + `config`/`proc_group`; `db`/`git_worktree`/`stage_engine`/`qg.checks`/`notifications` — лениво). + Исполняет ту же сюиту (`docker exec orchestrator-staging python3 .../staging_check.py --base-url + http://localhost:8501 --mode stub`, арги из config — ORCH-101) через `proc_group` (tree-kill, + таймаут `staging_runner_timeout_s=600`); маппит exit-код **единым** контрактом + `self_deploy.map_exit_code_to_status` (`0→SUCCESS`/иначе/None→`FAILED`; ORCH-061 infra-tolerance + остаётся внутри `staging_check.py`, раннер не пересуживает); пишет `15-staging-log.md` + (`author_agent: staging-runner`/`model_used: n/a`, 52c-схема) + best-effort push в **фичеветку** + (не в `main` — гейт читает worktree первым, усиливает AC-8); вызывает **существующий** + `advance_stage(current_stage="deploy-staging", finished_agent="deployer")` — без новых + рёбер/исходов (lease ORCH-114 берётся внутри `advance_stage`, раннер не трогает). +- **Двухуровневый исход (D5, анти-ORCH-110):** сюита **исполнилась** (реальный exit-код) → verdict→ + advance (FAILED → тот же откат `deploy-staging → development` + developer-retry, что у LLM); сюита + **не исполнилась** (tool-error: spawn-error/таймаут/`returncode None`) → инфра-сбой ≠ код-фейл → + bounded **DEFER** (re-queue `deployer`-джоба + restart-safe маркер `staging-runner infra-retry` в + `task_content`, без правки схемы БД), на исчерпании `staging_runner_infra_max_retries` → fail-closed + `FAILED` + advance + alert. Никогда тихий advance/ложный green; не клинит очередь; не жжёт + developer-retry на транзиентной инфре. +- **Флаги** (`config.py`, дефолт = боевое): `staging_runner_enabled` (kill-switch, env + `ORCH_STAGING_RUNNER_ENABLED`), `staging_runner_repos` (CSV; **пусто → self-hosting only**), + `staging_runner_timeout_s=600`, `staging_runner_infra_max_retries=2`, + `staging_runner_infra_retry_delay_s=30`. Откат = `ORCH_STAGING_RUNNER_ENABLED=false` → на + `deploy-staging` снова LLM-`deployer` через `_spawn` **байт-в-байт**. Наблюдаемость — in-process + счётчики + read-only блок `staging_runner` в `GET /queue` + один структурный лог-вердикт на прогон + (различает код-фейл и tool-error). Норматив сопровождения ORCH-118: обновлены + `llm-call-sites.md`/`llm-determinization-roadmap.md`/`llm-usage-policy.md` (A6 — реализован, машинные + блоки/инвариант «единственный транспорт S0» целы) + `deployer.md` (LLM-ветвь — fallback). Покрытие — + `tests/test_orch115_staging_runner.py` (TC-01…TC-13) + зелёные LLM-анти-дрейф тесты (TC-14). Детали — + `docs/work-items/ORCH-115/06-adr/ADR-001-deterministic-staging-runner.md`, сквозной + `docs/architecture/adr/adr-0048-deterministic-staging-runner.md`. + ## Машинный журнал уроков (ORCH-098) Шаг 1 («Фундамент», F2) эпика саморазвития: формализует свободнотекстовые «уроки» из `memory/` в **машинную структурированную таблицу отклонений конвейера** `lessons`, фундамент для будущих diff --git a/docs/architecture/llm-call-sites.md b/docs/architecture/llm-call-sites.md index aed8e95..4c361c2 100644 --- a/docs/architecture/llm-call-sites.md +++ b/docs/architecture/llm-call-sites.md @@ -58,7 +58,7 @@ | **A3** | `.openclaw/agents/developer.md` | стадия `development` | developer | код + PR | — | `check_ci_green:82` (+ `check_branch_mergeable:657`) — CI/merge | ~150–400k / 10–40 мин | да (через S0) | **P** | `keep-LLM` | написание кода — настоящее суждение; гейт судит CI/merge, не самоотчёт | | **A4** | `.openclaw/agents/reviewer.md` | стадия `review` | reviewer | `12-review.md` | `verdict:` | `check_reviewer_verdict:336` (`verdict:`) | ~100–300k / 5–25 мин | да (через S0) | **C** | `keep-LLM` | control path, но вердикт «приемлемость кода/решения» **НЕ деривируем** из exit-кода — настоящее суждение | | **A5** | `.openclaw/agents/tester.md` | стадия `testing` | tester | `13-test-report.md` | `result:` | `check_tests_passed:182` → `_parse_tests_verdict:226` (`result:`) | ~60–150k / 5–20 мин | да (через S0) | **C** | `needs-hybrid-fallback` | **avoidable**: PASS/FAIL = exit-code `pytest`+smoke (деривируем); LLM нужен лишь на триаж падений / маппинг TC↔критерии | -| **A6** | `.openclaw/agents/deployer.md` | стадии `deploy-staging` / `deploy` | deployer | `15-staging-log.md` / `14-deploy-log.md` | `staging_status:` / `deploy_status:` | `check_staging_status:599` → `_parse_staging_status:538` (`staging_status:`); `check_deploy_status:473` → `_parse_deploy_status:413` (`deploy_status:`) | ~40–120k / 3–15 мин | да (через S0) | **C** | `replace-deterministic-now` | **avoidable**: staging-вердикт = маппинг exit-кода `staging_check.py`; прод уже детерминирован Phase A/B/C (ORCH-036) | +| **A6** | `.openclaw/agents/deployer.md` | стадии `deploy-staging` / `deploy` | deployer | `15-staging-log.md` / `14-deploy-log.md` | `staging_status:` / `deploy_status:` | `check_staging_status:599` → `_parse_staging_status:538` (`staging_status:`); `check_deploy_status:473` → `_parse_deploy_status:413` (`deploy_status:`) | ~40–120k / 3–15 мин | да (через S0; для in-scope репо на `deploy-staging` — **нет**, перехват до `_spawn`) | **C** | `replace-deterministic-now` | **avoidable, СРЕЗ РЕАЛИЗОВАН (ORCH-115):** на `deploy-staging` для self-hosting `orchestrator` вердикт `staging_status:` производит детерминированный `src/staging_runner.py` (перехват в `launch_job` до `_spawn`, как D1/D2) — маппинг exit-кода `staging_check.py`; прод уже детерминирован Phase A/B/C (ORCH-036). LLM-ветвь остаётся fallback'ом под выключенным флагом / для не-self репо | | **D1** | `src/agents/launcher.py:389` (перехват в `launch_job` **до** `_spawn`; «Not an LLM spawn» `407`) | post-deploy edge | deploy-finalizer | jobs-row | — | — | — (детерминированный) | **нет** (слот, перехват до `_spawn`) | — | `already-deterministic` (эталон) | Занимает слот агента, но LLM не консультируется — рабочий прецедент замены | | **D2** | `src/agents/launcher.py:394` (перехват в `launch_job` **до** `_spawn`; «Not an LLM spawn» `428`) | post-deploy observation | post-deploy-monitor | jobs-row | — | — | — (детерминированный) | **нет** (слот, перехват до `_spawn`) | — | `already-deterministic` (эталон) | Тик наблюдения; LLM не консультируется | @@ -147,8 +147,14 @@ - **Docs + tests only (ORCH-118).** `STAGE_TRANSITIONS` / реестр и имена `QG_CHECKS`/`check_*` / machine-verdict-ключи (`verdict:`/`result:`/`staging_status:`/`deploy_status:`/`security_status:`/ - `coverage_status:`) / схема БД — **байт-в-байт не тронуты**. Раннеры замен **не** реализованы - (см. roadmap — follow-up'ы по роли, без Plane-ID). + `coverage_status:`) / схема БД — **байт-в-байт не тронуты** (это инвариант самой карты). +- **Реализованные срезы.** Первый срез roadmap'а — **deployer (staging-status)** — реализован + **ORCH-115** (`src/staging_runner.py`, перехват в `launch_job` до `_spawn`): на `deploy-staging` + для in-scope репо вердикт `staging_status:` производит детерминированный код, не LLM. Это + `replace-deterministic-now` без ввода второго транспорта (раннер LLM не зовёт) — карта/инвариант + «единственный транспорт S0» соблюдены. Машинный `ORCH-118-INVENTORY-BLOCK` сохраняет deployer как + `avoidable=yes`/`axis=C` (LLM-ветвь жива как fallback под выключенным флагом / для не-self репо). + Второй кандидат (`tester`) остаётся follow-up'ом по роли. - **Анти-дрейф тесты:** `tests/test_llm_call_site_inventory.py` (TC-01…TC-06, TC-09, TC-12, TC-13, TC-14) и `tests/test_llm_determinization_docs.py` (TC-07/08/11). Дискриминатор всех проверок — **«консультирует LLM», а не «спавнит subprocess»**. diff --git a/docs/architecture/llm-determinization-roadmap.md b/docs/architecture/llm-determinization-roadmap.md index 126b208..3f6579a 100644 --- a/docs/architecture/llm-determinization-roadmap.md +++ b/docs/architecture/llm-determinization-roadmap.md @@ -15,7 +15,17 @@ --- -## 1. Рекомендованный первый срез — **deployer (staging-status)** +## 1. Рекомендованный первый срез — **deployer (staging-status)** — ✅ РЕАЛИЗОВАН (ORCH-115) + +> **Статус: реализовано.** Срез выполнен в **ORCH-115** — `src/staging_runner.py` (перехват в +> `launch_job` до `_spawn`, как `D1`/`D2`): на стадии `deploy-staging` для self-hosting `orchestrator` +> вердикт `staging_status:` производит детерминированный код (маппинг exit-кода `staging_check.py` +> через `self_deploy.map_exit_code_to_status`), а не LLM. Под kill-switch `staging_runner_enabled` + +> скоуп `staging_runner_repos` (пусто → self-hosting only); LLM-ветвь остаётся fallback'ом. +> Контракт артефакта/гейта `check_staging_status`/`STAGE_TRANSITIONS`/схема БД — не тронуты. Детали — +> `docs/work-items/ORCH-115/06-adr/ADR-001-deterministic-staging-runner.md`, сквозной +> `docs/architecture/adr/adr-0048-deterministic-staging-runner.md`. Запись `rank 1` в машинном блоке +> §4 сохраняется (`first_slice = yes`, инвариант карты) как историческая фиксация первого среза. Обоснование (самый низкорисковый «чисто деривируемый» control path): diff --git a/docs/architecture/llm-usage-policy.md b/docs/architecture/llm-usage-policy.md index ceb0e89..c4b40a0 100644 --- a/docs/architecture/llm-usage-policy.md +++ b/docs/architecture/llm-usage-policy.md @@ -94,3 +94,8 @@ Call-site является **avoidable LLM control path** тогда и толь `launcher._spawn` (S0). Ввод второго транспорта (новый `_spawn`, импорт `anthropic`/`openai`/иного LLM-SDK, прямой HTTP Anthropic/Claude, второй model-invoking subprocess) запрещён без явного ADR; прибито тестами TC-01/TC-12. +- **Реализованный срез (ORCH-115).** Снятие C-консультации с деривируемым вердиктом — это разрешённое + `replace-deterministic-now`, а не ввод новой LLM-консультации. ORCH-115 снял A6/staging-status: + детерминированный `src/staging_runner.py` производит `staging_status:` без `_spawn` (перехват до + него, как `D1`/`D2`) — раннер **LLM не зовёт** и **второй транспорт не вводит**, поэтому инвариант + «единственный транспорт S0» соблюдён (TC-12 зелёный). Это образец для последующих срезов roadmap'а. diff --git a/docs/overview/tech-agents.md b/docs/overview/tech-agents.md index af5174e..7d919ed 100644 --- a/docs/overview/tech-agents.md +++ b/docs/overview/tech-agents.md @@ -48,6 +48,12 @@ Machine-verdict ключи читаются гейтами **только из Y Особенность: промпт `deployer` сознательно на английском (самый safety-critical — несёт запреты self-hosting в видной рамке); остальные пять — на русском. +Особенность (ORCH-115): на стадии `deploy-staging` для self-hosting `orchestrator` LLM-`deployer` +заменён **детерминированным staging-раннером** (`src/staging_runner.py`) — его работа была чисто +механической (запуск staging-сюиты + маппинг exit-кода). LLM-промпт `deployer` остаётся fallback'ом +под выключенным флагом / для не-self репо и продолжает вести прод-стадию `deploy`. Подробнее — +[конвейер](tech-pipeline.md) и [карта LLM-консультаций](../architecture/llm-call-sites.md). + ## Человек как седьмая роль Человек не пишет артефакты конвейера, но принимает два решения, которые не делегированы diff --git a/docs/overview/tech-pipeline.md b/docs/overview/tech-pipeline.md index f83dc42..a4d35e5 100644 --- a/docs/overview/tech-pipeline.md +++ b/docs/overview/tech-pipeline.md @@ -34,6 +34,16 @@ created → analysis → architecture → development → review → testing → | `done` | — | — | терминал | | `cancelled` | — | — | терминал (сток отмены) | +> **Детерминированный staging-раннер (ORCH-115).** На стадии `deploy-staging` для self-hosting +> `orchestrator` работу ведёт **детерминированный код** (`src/staging_runner.py`), а не LLM-агент +> `deployer`: он перехватывается в `launch_job` до запуска агента (как Phase A/B/C прод-деплоя), +> исполняет ту же staging-сюиту, маппит exit-код в `staging_status:` и инициирует **тот же** гейт +> `check_staging_status`. Это замена *продюсера* артефакта, а не гейта: контракт `15-staging-log.md`, +> имя/семантика `check_staging_status`, `STAGE_TRANSITIONS` — не изменились. Под kill-switch +> `staging_runner_enabled` (скоуп `staging_runner_repos`, пусто → self-hosting only); при выключении +> на стадии снова работает LLM-`deployer` байт-в-байт. Это первый реализованный срез +> determinization-roadmap (см. `docs/architecture/llm-determinization-roadmap.md`). + ## Под-гейты деплойного ребра — врезки, не стадии На переходе `deploy-staging → deploy` исполняются четыре под-гейта в нормативном порядке diff --git a/docs/overview/tech-quality-security.md b/docs/overview/tech-quality-security.md index 4a24573..abcae9c 100644 --- a/docs/overview/tech-quality-security.md +++ b/docs/overview/tech-quality-security.md @@ -50,7 +50,10 @@ LLM, и **нормативную политику** «LLM — только та control-path и его вердикт деривируем из exit-кодов (тогда консультацию можно заменить детерминированным раннером). Поимённо: avoidable = `{tester, deployer}`; настоящее суждение сохраняется у `{analyst, architect, developer, reviewer}`. Карта — снимок, прибитый структурными -анти-дрейф тестами; реализация замен — отдельные follow-up'ы по роли. +анти-дрейф тестами. **Первый срез реализован (ORCH-115):** на `deploy-staging` для self-hosting +`orchestrator` LLM-`deployer` заменён детерминированным `src/staging_runner.py` (вердикт +`staging_status:` = маппинг exit-кода staging-сюиты); LLM-ветвь остаётся fallback'ом, гейт +`check_staging_status` не тронут. Замена второго кандидата (`tester`) — follow-up по роли. - Карта вызовов LLM: [`../architecture/llm-call-sites.md`](../architecture/llm-call-sites.md) - Нормативная политика: [`../architecture/llm-usage-policy.md`](../architecture/llm-usage-policy.md) diff --git a/src/agents/launcher.py b/src/agents/launcher.py index c475642..015eefd 100644 --- a/src/agents/launcher.py +++ b/src/agents/launcher.py @@ -385,6 +385,14 @@ class AgentLauncher: (no-LLM) job — intercept it BEFORE _spawn (which would raise "Unknown agent", R-6) and run the deploy finalizer synchronously, driving the jobs row status itself. Returns None (no agent_run row). + + ORCH-115: the LLM ``deployer`` on the ``deploy-staging`` stage (self-hosting + scope) is replaced by a DETERMINISTIC staging-runner — intercepted here + BEFORE _spawn (same precedent as deploy-finalizer / post-deploy-monitor). The + discriminator is the TASK STAGE (deploy-staging), not the role name, so the + prod ``deploy`` deployer is never caught (staging_runner.should_intercept). + Kill-switch off / out of scope -> should_intercept False -> the prior LLM + deployer runs via _spawn byte-for-byte. """ if job.get("agent") == "deploy-finalizer": return self._run_deploy_finalizer_job(job) @@ -393,6 +401,11 @@ class AgentLauncher: # observation tick synchronously. Returns None (no agent_run row). if job.get("agent") == "post-deploy-monitor": return self._run_post_deploy_monitor_job(job) + # ORCH-115: deterministic staging-runner intercept (BEFORE _spawn). + if job.get("agent") == "deployer": + from .. import staging_runner + if staging_runner.should_intercept(job): + return self._run_staging_runner_job(job) return self._spawn( job["agent"], job["repo"], @@ -422,6 +435,28 @@ class AgentLauncher: pass return None + def _run_staging_runner_job(self, job: dict): + """ORCH-115: run the deterministic staging gate for a deployer job. + + Not an LLM spawn — there is no subprocess/monitor of an agent, so we mark the + jobs row done/failed here (mirror of _run_deploy_finalizer_job). The runner + never-raises, but we guard anyway so a runner fault can't wedge the worker. + Returns None (no agent_run row, _spawn not called). + """ + from ..db import mark_job + from .. import staging_runner + try: + staging_runner.run_staging_gate(job) + mark_job(job["id"], "done") + logger.info(f"staging-runner job {job['id']} done") + except Exception as e: + logger.error(f"staging-runner job {job['id']} failed: {e}") + try: + mark_job(job["id"], "failed", error=f"staging-runner error: {e}") + except Exception: + pass + return None + def _run_post_deploy_monitor_job(self, job: dict): """ORCH-021: run one deterministic post-deploy monitor tick for a job. diff --git a/src/config.py b/src/config.py index 888ef66..3db293d 100644 --- a/src/config.py +++ b/src/config.py @@ -413,6 +413,51 @@ class Settings(BaseSettings): coverage_tool_fail_closed: bool = False coverage_run_timeout_s: int = 900 + # ORCH-115: deterministic staging-runner replacing the LLM `deployer` agent on + # the `deploy-staging` stage for the self-hosting orchestrator. A new leaf + # src/staging_runner.py (never-raise) is intercepted in launch_job BEFORE _spawn + # (mirroring the deploy-finalizer / post-deploy-monitor reserved-agent + # precedent, launcher.py:389/394): it runs the SAME staging suite the LLM ran + # (`docker exec orchestrator-staging python3 .../staging_check.py`), maps the + # exit-code -> staging_status: via the existing self_deploy.map_exit_code_to_status + # contract, writes 15-staging-log.md, and initiates the EXISTING check_staging_status + # gate exactly as a finished LLM-deployer would. The artifact contract, the gate, + # STAGE_TRANSITIONS and the DB schema are byte-for-byte UNCHANGED — this only + # replaces the *producer* of the artifact. Pattern = coverage_gate_* / self_deploy_*. + # See docs/work-items/ORCH-115/06-adr/ADR-001-deterministic-staging-runner.md and + # docs/architecture/adr/adr-0048-deterministic-staging-runner.md. + # staging_runner_enabled -> SINGLE kill-switch (env + # ORCH_STAGING_RUNNER_ENABLED). False -> the + # intercept never fires -> the prior LLM + # deployer runs on deploy-staging via _spawn + # byte-for-byte as before ORCH-115 (D8/AC-6). + # staging_runner_repos -> CSV scope (env ORCH_STAGING_RUNNER_REPOS). + # Empty -> self-hosting only (orchestrator) + # via is_self_hosting_repo; non-empty -> + # membership. Mirrors coverage_gate_repos. + # staging_runner_timeout_s -> wall-clock budget for the docker-exec + # staging suite (env ORCH_STAGING_RUNNER_TIMEOUT_S). + # Malformed/non-positive -> default + WARNING + # (never-break). Aligned with the cross-cutting + # budget invariant ORCH-065/109/110 WITHOUT + # touching reaper_max_running_s (D9): it replaces + # the up-to-900s LLM staging window with a bounded + # <=600s deterministic one (Σ on the edge does not grow). + # staging_runner_infra_max_retries -> tool-error (suite did NOT execute: spawn-error / + # timeout / returncode None) bounded DEFER budget + # before a fail-closed FAILED (env + # ORCH_STAGING_RUNNER_INFRA_MAX_RETRIES). Mirrors + # merge_retest_infra_max_retries — infra hiccup is + # NOT a code-fault, so it never burns a developer-retry + # until the budget is exhausted (D5, anti ORCH-110). + # staging_runner_infra_retry_delay_s-> delay before the re-queued deployer job + # (env ORCH_STAGING_RUNNER_INFRA_RETRY_DELAY_S). + staging_runner_enabled: bool = True + staging_runner_repos: str = "" + staging_runner_timeout_s: int = 600 + staging_runner_infra_max_retries: int = 2 + staging_runner_infra_retry_delay_s: int = 30 + # ORCH-098 (FND/F2): machine lessons-journal — additive `lessons` table + leaf # src/lessons.py (never-raise observer, by образцу serial_gate/coverage_gate/ # metrics). The journal is an OBSERVER, never a Quality Gate: writing a lesson diff --git a/src/main.py b/src/main.py index 5790877..3877bbd 100644 --- a/src/main.py +++ b/src/main.py @@ -235,6 +235,7 @@ async def queue(): from . import lessons from . import checkout_hygiene from . import transition_lease + from . import staging_runner from .disk_watchdog import disk_watchdog from .build_cache_pruner import build_cache_pruner return { @@ -283,6 +284,11 @@ async def queue(): # (owner/stage/age/live) + defer/reclaim/CAS-lost counters. Additive block; # never-raise. "transition_lease": transition_lease.snapshot(), + # ORCH-115 (FR-7 / AC-10): deterministic staging-runner observability + # (read-only) — kill-switch, scope, timeout/infra budget + run/success/ + # failed/tool_error/deferred counters, so a code-fail FAILED is distinguishable + # from an infra tool-error. Additive block; never-raise. + "staging_runner": staging_runner.snapshot(), # ORCH-098 (FR-4 / AC-4): lessons-journal observability (read-only) — # kill-switch + counts by type/status + last N lessons. Additive block; # never-raise (snapshot() returns {"enabled": ...} minimum on error). diff --git a/src/staging_runner.py b/src/staging_runner.py new file mode 100644 index 0000000..296bc75 --- /dev/null +++ b/src/staging_runner.py @@ -0,0 +1,581 @@ +"""Deterministic staging-runner (ORCH-115). + +The `deploy-staging` stage for the self-hosting ``orchestrator`` repo was driven by +the LLM ``deployer`` agent, but the actual work is purely deterministic +(``.openclaw/agents/deployer.md`` steps 1-4): run the staging suite, map its +**exit-code** to a verdict (``0 -> SUCCESS``, else ``FAILED``), write +``15-staging-log.md`` and commit it. This leaf replaces that LLM consultation with +deterministic code, intercepted in ``launcher.launch_job`` BEFORE ``_spawn`` (the +``deploy-finalizer`` / ``post-deploy-monitor`` reserved-agent precedent, +``launcher.py:389/394``). + +What is and is NOT changed (NFR-1, the critical invariant): + * UNCHANGED — the artifact contract (``15-staging-log.md`` with + ``staging_status: SUCCESS|FAILED``), the gate ``check_staging_status`` / + ``_parse_staging_status``, ``STAGE_TRANSITIONS``, the DB schema. This module + replaces only the *producer* of the artifact, never the gate that reads it. + * NEW — a deterministic producer + a launch_job intercept. Under a kill-switch + + repo-scope CSV; fail-safe back to the LLM path when off / out of scope. + +This module is a **leaf** (mirror of ``self_deploy`` / ``proc_group`` / +``staging_verdict``): it imports only ``config`` / ``logging`` / ``proc_group`` at +module load; ``db`` / ``git_worktree`` / ``self_deploy.map_exit_code_to_status`` / +``qg.checks`` / ``stage_engine.advance_stage`` / ``notifications`` are imported +LAZILY inside functions so the heavy ``stage_engine`` is never pulled at import and +no import cycle forms (pattern: ``transition_lease`` / ``merge_gate``). Every public +function honours a **never-raise** contract (AC-7): a staging-infra hiccup can never +crash the worker / wedge the queue. + +Two-level outcome (D5 — the key safety decision, anti ORCH-110): + * the suite EXECUTED (a real exit-code, 0 or non-zero) -> trust the code: + ``0 -> SUCCESS -> advance``; ``!=0 -> FAILED -> the existing rollback + deploy-staging -> development`` (same developer-retry path as a FAILED LLM + verdict). ORCH-061 infra-tolerance is already INSIDE ``staging_check.py`` — the + runner never re-judges it (BR-4). + * the suite did NOT execute (tool-error: spawn-error / timeout / ``returncode is + None``) -> an infra fault, NOT a code fault -> a bounded DEFER (re-queue a fresh + ``deployer`` job with a delay + a restart-safe marker). On budget exhaustion -> + fail-closed ``FAILED`` + advance + alert. So the runner NEVER does a silent + advance / false green, and NEVER wedges the queue, but does NOT burn a + developer-retry on transient infra. +""" + +import logging +import time + +from .config import settings +from . import proc_group + +logger = logging.getLogger("orchestrator.staging_runner") + +# Platform service literal — the staging compose service the suite runs inside. +# Already an accepted platform literal (mirror image_freshness._STAGING_SERVICE); +# NOT a host hardcode (test_no_host_hardcodes forbids host IP/home/hostname only). +STAGING_SERVICE = "orchestrator-staging" + +# Default wall-clock budget for the docker-exec staging suite (D9). Kept <= the LLM +# staging window it replaces so Σ(work on the deploy-staging edge) does not grow and +# the cross-cutting reaper invariant (ORCH-065/109/110) holds WITHOUT touching +# reaper_max_running_s. +_DEFAULT_TIMEOUT_S = 600 + +_GIT_TIMEOUT = 60 + +# Restart-safe DEFER marker (counted from the persisted jobs queue, mirror of +# stage_engine._merge_infra_retry_count). Embedded verbatim in the re-queued job's +# task_content so a service restart never resets the infra-retry budget. +_INFRA_RETRY_MARKER = "staging-runner infra-retry" + +# In-process observability counters (mirror merge_gate._MERGE_GATE_COUNTERS, ORCH-110). +_STAGING_RUNNER_COUNTERS: dict = { + "runs": 0, # run_staging_gate entered + "success": 0, # suite ran, exit 0 -> SUCCESS + "failed": 0, # suite ran non-zero, OR infra budget exhausted -> FAILED + "tool_error": 0, # suite did NOT execute (spawn-error / timeout / None) + "deferred": 0, # bounded infra DEFER (re-queued) +} + + +def _bump(key: str) -> None: + """Increment an observability counter. Never raises.""" + try: + _STAGING_RUNNER_COUNTERS[key] += 1 + except Exception: # noqa: BLE001 - observability must never break a decision + pass + + +# --------------------------------------------------------------------------- +# Conditionality (D8 / FR-6 / AC-6) +# --------------------------------------------------------------------------- +def applies(repo: str) -> bool: + """Whether the deterministic staging-runner is REAL for ``repo``. + + Mirrors ``self_deploy_applies`` / ``coverage_gate``: + * ``staging_runner_enabled=False`` -> always False (global kill-switch); the + legacy LLM-deployer path runs on deploy-staging via ``_spawn``. + * ``staging_runner_repos`` (CSV) non-empty -> only the listed repos. + * empty CSV -> only the self-hosting repo (``orchestrator``), which is the only + one with a staging instance. + Checked FIRST in ``should_intercept`` (local, no network, no DB) so a disabled + flag costs nothing. never-raise -> False (fail-safe to the prior LLM path). + """ + try: + if not settings.staging_runner_enabled: + return False + raw = (settings.staging_runner_repos or "").strip() + if raw: + allowed = {r.strip().lower() for r in raw.split(",") if r.strip()} + return (repo or "").strip().lower() in allowed + # Lazy import keeps this module a leaf (no qg import at module load). + from .qg.checks import is_self_hosting_repo + return is_self_hosting_repo(repo) + except Exception as e: # noqa: BLE001 - never-raise contract + logger.warning("staging_runner.applies error for %s: %s", repo, e) + return False + + +def should_intercept(job: dict) -> bool: + """True iff this ``deployer`` job is the deterministic staging-suite job (D1). + + The discriminator of "staging vs prod" is the TASK STAGE, not the role name + (``deployer`` owns both ``deploy-staging`` and ``deploy``): intercept iff + ``agent == "deployer"`` AND the task's ``tasks.stage == "deploy-staging"`` AND + ``applies(repo)``. For self-hosting the prod ``deploy`` edge runs Phase A (no + deployer spawn); for non-self repos ``applies`` is False, so a non-self prod + ``deployer`` job is never intercepted (R-1 / TC-06). + + never-raise -> False (a DB-lookup failure falls through to ``_spawn``, fail-safe + to the prior LLM path). + """ + try: + if (job.get("agent") or "") != "deployer": + return False + # applies() FIRST (local, no DB): disabled flag -> zero DB overhead. + if not applies(job.get("repo")): + return False + task_id = job.get("task_id") + if task_id is None: + return False + from .db import get_db + conn = get_db() + row = conn.execute("SELECT stage FROM tasks WHERE id=?", (task_id,)).fetchone() + conn.close() + if not row: + return False + return (row[0] or "") == "deploy-staging" + except Exception as e: # noqa: BLE001 - never-raise contract + logger.warning("staging_runner.should_intercept error: %s", e) + return False + + +# --------------------------------------------------------------------------- +# Suite execution (D3 / FR-2 / NFR-3 / AC-8 / AC-9) +# --------------------------------------------------------------------------- +def build_staging_command() -> list[str]: + """Build the canonical staging-suite argv (same command the LLM-deployer ran). + + ``docker exec python3 //scripts/staging_check.py + --base-url http://localhost: --mode stub``. Host-specifics come from + config (ORCH-101, no host hardcodes). Self-hosting safety (BR-7 / AC-8 / TC-12): + NO restart of 8500, NO ``docker compose up orchestrator`` / ``--build``, NO + force-push, NO ``.env`` edit — the runner only reads/executes the staging suite + (8501) and writes a log. + """ + from .qg.checks import SELF_HOSTING_REPO + repos_dir = (settings.repos_dir or "/repos").rstrip("/") + script = f"{repos_dir}/{SELF_HOSTING_REPO}/scripts/staging_check.py" + base_url = f"http://localhost:{int(settings.staging_port)}" + return [ + "docker", "exec", STAGING_SERVICE, + "python3", script, + "--base-url", base_url, + "--mode", "stub", + ] + + +def _resolve_timeout() -> int: + """Resolve ``staging_runner_timeout_s`` (malformed/non-positive -> default + + WARNING, never-break — mirror of ``merge_gate._resolve_retest_timeout``).""" + raw = getattr(settings, "staging_runner_timeout_s", _DEFAULT_TIMEOUT_S) + try: + t = int(raw) + if t > 0: + return t + logger.warning( + "staging_runner_timeout_s non-positive (%r) -> default %ds", + raw, _DEFAULT_TIMEOUT_S, + ) + except (TypeError, ValueError): + logger.warning( + "staging_runner_timeout_s malformed (%r) -> default %ds", + raw, _DEFAULT_TIMEOUT_S, + ) + return _DEFAULT_TIMEOUT_S + + +def run_staging_suite() -> proc_group.ProcResult: + """Execute the staging suite in its own process group, tree-killed on timeout. + + Reuses ``proc_group.run_in_process_group`` (ORCH-110) so a hung docker-exec / + pytest subtree is killed whole (no orphans, AC-9). Never raises (proc_group + degrades any OS error to a safe ``ProcResult``). + """ + cmd = build_staging_command() + timeout = _resolve_timeout() + try: + grace = float(getattr(settings, "agent_kill_grace_seconds", 5) or 5) + except (TypeError, ValueError): + grace = 5.0 + return proc_group.run_in_process_group( + cmd, + cwd=None, + timeout=timeout, + grace_s=grace, + tree_kill=bool(getattr(settings, "subprocess_tree_kill_enabled", True)), + ) + + +# --------------------------------------------------------------------------- +# exit-code -> verdict (D4 / FR-3 / AC-3) — reuse the single contract, no 2nd mapping +# --------------------------------------------------------------------------- +def map_exit_code_to_status(exit_code) -> str: + """``0 -> SUCCESS``; non-zero / None / non-int -> ``FAILED`` (fail-closed). + + Delegates to ``self_deploy.map_exit_code_to_status`` — the SAME pure, unit-tested + contract the deploy-finalizer uses (BR-4: no second, drifting mapping). + """ + from .self_deploy import map_exit_code_to_status as _map + return _map(exit_code) + + +# --------------------------------------------------------------------------- +# Artifact 15-staging-log.md (D6 / FR-4 / AC-2 / AC-8) — mirror write_deploy_log +# --------------------------------------------------------------------------- +def _extract_infra_waived(stdout: str) -> list[str]: + """Lines from the suite stdout carrying the ORCH-061 ``INFRA-WAIVED:`` marker + (copied into the log body for observability, as the prompt required).""" + if not stdout: + return [] + return [ln.strip() for ln in stdout.splitlines() if "INFRA-WAIVED" in ln] + + +def build_staging_log( + work_item_id: str, exit_code, status: str, stdout: str = "", *, tool_error: bool = False +) -> str: + """Render a ``15-staging-log.md`` body whose ``staging_status:`` frontmatter is the + verdict ``check_staging_status`` / ``_parse_staging_status`` reads (contract + UNCHANGED, AC-2). Carries the mandatory 52c schema (``work_item`` / ``stage`` / + ``author_agent`` / ``status`` / ``created_at`` / ``model_used``); ``author_agent: + staging-runner`` / ``model_used: n/a`` honestly reflect the DETERMINISTIC producer. + The machine key ``staging_status:`` and its UPPERCASE value are NOT changed. + + Written as a literal block (mirror of ``self_deploy.build_deploy_log``) so the + machine-read frontmatter is byte-exact; only the frontmatter is machine-read, the + body is informational. + """ + import datetime + created = datetime.date.today().isoformat() + sub_status = "success" if status == "SUCCESS" else "failed" + base_url = f"http://localhost:{int(settings.staging_port)}" + + waived = _extract_infra_waived(stdout) + waived_body = "" + if waived: + waived_body = "\nINFRA-WAIVED lines (ORCH-061, copied for observability):\n" + "\n".join( + f"- {ln}" for ln in waived + ) + "\n" + + tail = "" + if stdout: + tail_text = stdout.strip()[-1500:] + if tail_text: + tail = f"\nStaging suite stdout (tail):\n```\n{tail_text}\n```\n" + + note = ( + "Staging suite did NOT execute (tool-error) and the infra-retry budget was " + "exhausted -> fail-closed FAILED." + if tool_error + else f"Staging suite exit-code `{exit_code}` -> `staging_status: {status}`." + ) + + return ( + "---\n" + f"staging_status: {status}\n" + f"work_item: {work_item_id}\n" + "stage: deploy-staging\n" + "author_agent: staging-runner\n" + f"status: {sub_status}\n" + f"created_at: {created}\n" + "model_used: n/a\n" + f"exit_code: {exit_code}\n" + f"base_url: {base_url}\n" + "---\n\n" + "# Staging Gate Log (deterministic runner, ORCH-115)\n\n" + f"{note}\n\n" + "Вердикт зафиксирован детерминированным staging-раннером (ORCH-115), не LLM. " + "infra-tolerance (ORCH-061) уже учтена внутри `staging_check.py` — раннер её не " + "пересуживает.\n" + f"{waived_body}" + f"{tail}" + ) + + +def write_staging_log( + repo: str, work_item_id: str, branch: str, exit_code, status: str, + stdout: str = "", *, tool_error: bool = False, +) -> bool: + """Write ``15-staging-log.md`` into the task worktree (so ``check_staging_status`` + reads it) and best-effort commit+push it to the FEATURE BRANCH. Returns True iff + the file was written. Never raises. + + Mirror of ``self_deploy.write_deploy_log`` EXCEPT: the actor name is + ``staging-runner`` and the log is pushed only to the feature branch — there is NO + separate PR-merge of the log into ``main`` (the gate reads the worktree first; + excluding any direct work on ``main`` strengthens AC-8 / BR-7). The feature branch + is merged into ``main`` later by the normal merge-gate path. + """ + import os + import subprocess + from .git_worktree import get_worktree_path + + rel = f"docs/work-items/{work_item_id}/15-staging-log.md" + try: + wt = get_worktree_path(repo, branch) + except Exception as e: # noqa: BLE001 - never-raise + logger.error("write_staging_log: worktree error for %s/%s: %s", repo, branch, e) + return False + + path = os.path.join(wt, rel) + content = build_staging_log(work_item_id, exit_code, status, stdout, tool_error=tool_error) + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(content) + except OSError as e: + logger.error("write_staging_log: write error at %s: %s", path, e) + return False + + # Best-effort commit + push to the feature branch (the gate also falls back to + # origin/main). ORCH-101: HOME + email domain from Settings; the actor NAME stays + # the platform literal `staging-runner` (deterministic system-actor commits). + _email = f"staging-runner@{settings.git_email_domain}" + git_env = { + **os.environ, + "HOME": settings.agent_home_dir, + "GIT_AUTHOR_NAME": "staging-runner", + "GIT_AUTHOR_EMAIL": _email, + "GIT_COMMITTER_NAME": "staging-runner", + "GIT_COMMITTER_EMAIL": _email, + } + try: + subprocess.run(["git", "-C", wt, "add", rel], + capture_output=True, timeout=_GIT_TIMEOUT, env=git_env) + commit = subprocess.run( + ["git", "-C", wt, "commit", "-m", + f"staging(ORCH-115): staging gate {status} for {work_item_id}"], + capture_output=True, text=True, timeout=_GIT_TIMEOUT, env=git_env, + ) + if commit.returncode == 0: + subprocess.run(["git", "-C", wt, "push", "origin", branch], + capture_output=True, timeout=_GIT_TIMEOUT, env=git_env) + except (subprocess.SubprocessError, OSError) as e: + logger.warning("write_staging_log: git commit/push best-effort failed: %s", e) + return True + + +# --------------------------------------------------------------------------- +# Existing-gate initiation (D7 / FR-5 / AC-4) — no new routing branch +# --------------------------------------------------------------------------- +def _advance(task_id, repo: str, work_item_id: str, branch: str) -> None: + """Initiate the SAME ``advance_stage`` evaluation a finished LLM-deployer would + (``finished_agent="deployer"`` on ``deploy-staging``): SUCCESS -> sub-gates + security->merge->coverage->image-freshness (ORCH-022/043/027/058) + Phase A + (ORCH-036); FAILED -> the existing rollback deploy-staging -> development + (``stage_engine.py:932``). No new routing branch. The transition-lease (ORCH-114) + is taken INSIDE advance_stage on the side-effectful edge — the runner never + touches it (task boundary O1). never-raise.""" + try: + from . import stage_engine + stage_engine.advance_stage( + task_id=task_id, + current_stage="deploy-staging", + repo=repo, + work_item_id=work_item_id, + branch=branch, + finished_agent="deployer", + ) + except Exception as e: # noqa: BLE001 - never-raise into the worker + logger.error( + "staging_runner._advance: advance_stage failed for task %s (%s): %s", + task_id, work_item_id, e, + ) + + +# --------------------------------------------------------------------------- +# Two-level outcome (D5) — tool-error DEFER bookkeeping +# --------------------------------------------------------------------------- +def _infra_retry_count(task_id) -> int: + """How many times this task was re-queued by the tool-error DEFER path + (restart-safe; counted from the persisted jobs queue by the marker — mirror of + ``stage_engine._merge_infra_retry_count``). Never raises -> 0 on error.""" + try: + from .db import get_db + conn = get_db() + n = conn.execute( + "SELECT COUNT(*) FROM jobs WHERE task_id=? AND task_content LIKE ?", + (task_id, f"%{_INFRA_RETRY_MARKER}%"), + ).fetchone()[0] + conn.close() + return int(n) + except Exception as e: # noqa: BLE001 - never-raise + logger.warning("staging_runner._infra_retry_count error for %s: %s", task_id, e) + return 0 + + +def _handle_tool_error( + task_id, repo: str, work_item_id: str, branch: str, result: proc_group.ProcResult +) -> None: + """Suite did NOT execute (tool-error) -> bounded DEFER, then fail-closed (D5). + + Anti ORCH-110: an infra fault is NOT a code fault, so we re-queue a fresh + ``deployer`` job (which re-enters this runner) with a delay instead of an + immediate FAILED-rollback that would burn a developer-retry. On budget exhaustion + -> write ``staging_status: FAILED`` + advance (the existing rollback) + an + INFRA-specific alert (explicitly "not a code defect"). Never a silent advance / + false green; never wedges the queue. never-raise.""" + retries = _infra_retry_count(task_id) + try: + max_retries = int(settings.staging_runner_infra_max_retries) + except (TypeError, ValueError): + max_retries = 2 + try: + delay = int(settings.staging_runner_infra_retry_delay_s) + except (TypeError, ValueError): + delay = 30 + + if retries < max_retries: + _bump("deferred") + reason = "timeout" if result.timed_out else "suite did not execute (tool-error)" + task_desc = ( + f"Work item: {work_item_id}\nRepo: {repo}\nBranch: {branch}\n" + f"Stage: deploy-staging\nNote: {_INFRA_RETRY_MARKER} " + f"(attempt {retries + 1}/{max_retries}) — {reason}, retrying after {delay}s." + ) + try: + from .db import enqueue_job + new_job = enqueue_job( + "deployer", repo, task_desc, task_id=task_id, available_at_delay_s=delay, + ) + logger.warning( + "Task %s (%s): staging suite did not execute (%s) -> infra-DEFER " + "(job_id=%s, attempt %d/%d)", + task_id, work_item_id, reason, new_job, retries + 1, max_retries, + ) + except Exception as e: # noqa: BLE001 - never-raise + logger.error("staging_runner: infra-DEFER enqueue failed for %s: %s", task_id, e) + return + + # Budget exhausted -> fail-closed FAILED (terminal, never a false green). + _bump("failed") + logger.error( + "Task %s (%s): staging tool-error DEFER budget exhausted (%d) -> fail-closed FAILED", + task_id, work_item_id, max_retries, + ) + write_staging_log(repo, work_item_id, branch, result.returncode, "FAILED", + result.stdout, tool_error=True) + _alert_infra_exhausted(work_item_id, max_retries) + _advance(task_id, repo, work_item_id, branch) + + +def _alert_infra_exhausted(work_item_id: str, max_retries: int) -> None: + """Best-effort Telegram alert that the staging suite never executed (infra, NOT a + code defect) after the retry budget. never-raise.""" + try: + from .notifications import send_telegram, link_for + send_telegram( + f"\U0001f6a8 {link_for(work_item_id)}: staging suite не запустилась " + f"(инфра, НЕ дефект кода) после {max_retries} попыток — fail-closed FAILED, " + f"откат на development. Нужно проверить staging-стенд." + ) + except Exception as e: # noqa: BLE001 - never-raise + logger.warning("staging_runner: infra-exhausted alert failed for %s: %s", work_item_id, e) + + +# --------------------------------------------------------------------------- +# Entry point (D2) — owns the full deterministic flow, mirror run_deploy_finalizer +# --------------------------------------------------------------------------- +def run_staging_gate(job: dict) -> None: + """Deterministic staging gate for a ``deployer`` job on ``deploy-staging``. + + Flow (mirror of ``stage_engine.run_deploy_finalizer``): + 1. resolve ``work_item_id`` / ``branch`` by ``task_id``; + 2. execute the staging suite (D3) -> ProcResult; + 3. suite EXECUTED -> map exit-code -> ``staging_status:``, write + ``15-staging-log.md``, initiate the existing gate via ``advance_stage`` (D7); + 4. suite did NOT execute (tool-error) -> bounded DEFER / fail-closed (D5); + 5. observability counters + one structured verdict log (D10). + Never raises into the caller (the launcher marks the job done/failed).""" + started = time.time() + _bump("runs") + task_id = job.get("task_id") + repo = job.get("repo") + + # 1. resolve task fields. + work_item_id, branch = None, None + try: + from .db import get_db + conn = get_db() + row = conn.execute( + "SELECT work_item_id, branch FROM tasks WHERE id=?", (task_id,) + ).fetchone() + conn.close() + if row: + work_item_id, branch = row[0], row[1] + except Exception as e: # noqa: BLE001 - never-raise + logger.error("staging_runner: task lookup failed for task_id=%s: %s", task_id, e) + if not work_item_id or not branch: + logger.error( + "staging_runner: missing work_item_id/branch for task_id=%s — aborting", task_id + ) + return + + # 2-4. execute + classify + route — guarded so AC-7 (never-raise) holds even if + # an unexpected error escapes a sub-step (the worker must never crash on staging + # infra; the task is left on deploy-staging for the reconciler/reaper to re-drive). + try: + result = run_staging_suite() + duration_s = round(time.time() - started, 1) + suite_ran = (result.returncode is not None) and (not result.timed_out) + + if suite_ran: + # 3. trust the exit-code (ORCH-061 already inside staging_check.py). + status = map_exit_code_to_status(result.returncode) + _bump("success" if status == "SUCCESS" else "failed") + logger.info( + "staging_runner verdict: work_item=%s repo=%s exit_code=%s status=%s " + "duration_s=%s outcome=%s", + work_item_id, repo, result.returncode, status, duration_s, + "code-pass" if status == "SUCCESS" else "code-fail", + ) + write_staging_log(repo, work_item_id, branch, result.returncode, status, result.stdout) + _advance(task_id, repo, work_item_id, branch) + return + + # 4. tool-error (suite did not execute) -> DEFER / fail-closed (D5). + _bump("tool_error") + logger.warning( + "staging_runner verdict: work_item=%s repo=%s exit_code=%s status=%s " + "duration_s=%s outcome=tool-error (timed_out=%s)", + work_item_id, repo, result.returncode, "TOOL-ERROR", duration_s, result.timed_out, + ) + _handle_tool_error(task_id, repo, work_item_id, branch, result) + except Exception as e: # noqa: BLE001 - never-raise into the worker (AC-7) + logger.error( + "staging_runner.run_staging_gate: unexpected error for task %s (%s): %s", + task_id, work_item_id, e, + ) + + +# --------------------------------------------------------------------------- +# Observability (D10 / FR-7 / AC-10) +# --------------------------------------------------------------------------- +def snapshot() -> dict: + """Read-only staging-runner summary for ``GET /queue`` (FR-7 / AC-10). + + Additive block; existing ``/queue`` keys are untouched. never-raise: any error -> + a minimal dict with the kill-switch state.""" + try: + return { + "enabled": bool(settings.staging_runner_enabled), + "repos": getattr(settings, "staging_runner_repos", "") or "", + "timeout_s": getattr(settings, "staging_runner_timeout_s", _DEFAULT_TIMEOUT_S), + "infra_max_retries": getattr(settings, "staging_runner_infra_max_retries", 2), + "runs": _STAGING_RUNNER_COUNTERS["runs"], + "success": _STAGING_RUNNER_COUNTERS["success"], + "failed": _STAGING_RUNNER_COUNTERS["failed"], + "tool_error": _STAGING_RUNNER_COUNTERS["tool_error"], + "deferred": _STAGING_RUNNER_COUNTERS["deferred"], + } + except Exception as e: # noqa: BLE001 - never-raise -> minimal dict + logger.warning("staging_runner.snapshot error: %s", e) + return {"enabled": False} diff --git a/tests/test_orch115_staging_runner.py b/tests/test_orch115_staging_runner.py new file mode 100644 index 0000000..da8c415 --- /dev/null +++ b/tests/test_orch115_staging_runner.py @@ -0,0 +1,438 @@ +"""ORCH-115 — deterministic staging-runner replacing the LLM deployer on +`deploy-staging` (self-hosting orchestrator). + +Covers Phase 1 (04-test-plan.yaml TC-01…TC-13): the launch_job intercept BEFORE +_spawn, the exit-code -> staging_status: mapping, 15-staging-log.md render + the +UNCHANGED gate contract, advance_stage initiation, kill-switch/scope, never-raise / +fail-safe, the bounded tool-error DEFER, process timeout, self-hosting safety, the +anti-drift invariant (STAGE_TRANSITIONS / QG_CHECKS / DB schema untouched), and the +/queue observability block. + +No live Claude CLI, docker or network: the staging subprocess and advance_stage are +mocked; the pure mapping/render is tested directly. (TC-14 — the LLM call-site map +anti-drift — lives in tests/test_llm_call_site_inventory.py / test_llm_determinization_docs.py.) +""" +import os +import tempfile + +import pytest + +_test_db = os.path.join(tempfile.gettempdir(), "test_orch115_staging_runner.db") +os.environ["ORCH_DB_PATH"] = _test_db +os.environ["ORCH_REPOS_DIR"] = tempfile.gettempdir() +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +from unittest.mock import MagicMock # noqa: E402 + +import src.db as _db # noqa: E402 +from src.db import init_db, get_db, enqueue_job # noqa: E402 +from src import staging_runner # noqa: E402 +from src import stage_engine # noqa: E402 +from src import config as cfg # noqa: E402 +from src.proc_group import ProcResult # noqa: E402 +from src.agents.launcher import AgentLauncher # noqa: E402 + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def fresh_db(monkeypatch, tmp_path): + monkeypatch.setattr(_db.settings, "db_path", _test_db) + if os.path.exists(_test_db): + os.unlink(_test_db) + init_db() + # Worktree artefacts land in tmp; git commit/push degrade gracefully (no repo). + monkeypatch.setattr("src.git_worktree.settings.worktrees_dir", str(tmp_path), raising=False) + # Runner ON, self-hosting scope (default). + monkeypatch.setattr(cfg.settings, "staging_runner_enabled", True, raising=False) + monkeypatch.setattr(cfg.settings, "staging_runner_repos", "", raising=False) + monkeypatch.setattr(cfg.settings, "staging_runner_infra_max_retries", 2, raising=False) + # Reset in-process counters between tests. + for k in staging_runner._STAGING_RUNNER_COUNTERS: + staging_runner._STAGING_RUNNER_COUNTERS[k] = 0 + yield + + +def _make_task(stage, repo="orchestrator", wi="ORCH-115", branch="feature/ORCH-115-x"): + conn = get_db() + cur = conn.execute( + "INSERT INTO tasks (plane_id, work_item_id, repo, branch, stage) VALUES (?, ?, ?, ?, ?)", + (f"plane-{wi}", wi, repo, branch, stage), + ) + tid = cur.lastrowid + conn.commit() + conn.close() + return tid + + +def _make_job(agent, repo, task_id, content="x"): + return enqueue_job(agent, repo, content, task_id=task_id) + + +def _job_dict(job_id, agent, repo, task_id): + return {"id": job_id, "agent": agent, "repo": repo, "task_id": task_id, "task_content": "x"} + + +def _read_log(tmp_path, repo, branch, wi): + from src.git_worktree import get_worktree_path + p = os.path.join(get_worktree_path(repo, branch), f"docs/work-items/{wi}/15-staging-log.md") + with open(p, "r", encoding="utf-8") as f: + return f.read() + + +# --------------------------------------------------------------------------- +# TC-01 — applies(): kill-switch / scope / fail-safe (FR-6 / AC-6) +# --------------------------------------------------------------------------- +def test_tc01_applies_killswitch_and_scope(monkeypatch): + # enabled=False -> False (fall back to the LLM path) regardless of repo. + monkeypatch.setattr(cfg.settings, "staging_runner_enabled", False) + assert staging_runner.applies("orchestrator") is False + + # enabled, empty CSV -> self-hosting only. + monkeypatch.setattr(cfg.settings, "staging_runner_enabled", True) + monkeypatch.setattr(cfg.settings, "staging_runner_repos", "") + assert staging_runner.applies("orchestrator") is True + assert staging_runner.applies("enduro-trails") is False + assert staging_runner.applies("") is False + + # non-empty CSV -> membership (case-insensitive). + monkeypatch.setattr(cfg.settings, "staging_runner_repos", "enduro-trails, orchestrator") + assert staging_runner.applies("ENDURO-TRAILS") is True + assert staging_runner.applies("other-repo") is False + + +def test_tc01_applies_never_raises(monkeypatch): + # A settings attribute that explodes on read -> applies() degrades to False. + class Boom: + @property + def staging_runner_enabled(self): + raise RuntimeError("boom") + monkeypatch.setattr(staging_runner, "settings", Boom()) + assert staging_runner.applies("orchestrator") is False + + +# --------------------------------------------------------------------------- +# TC-02 — exit-code -> verdict, single shared contract (FR-3 / AC-3) +# --------------------------------------------------------------------------- +def test_tc02_map_exit_code(): + from src import self_deploy + assert staging_runner.map_exit_code_to_status(0) == "SUCCESS" + for bad in (1, 2, 137, -9): + assert staging_runner.map_exit_code_to_status(bad) == "FAILED" + for noncode in (None, "x", object()): + assert staging_runner.map_exit_code_to_status(noncode) == "FAILED" + # Same contract as the deploy-finalizer (no second, drifting mapping). + for v in (0, 1, None, "garbage"): + assert staging_runner.map_exit_code_to_status(v) == self_deploy.map_exit_code_to_status(v) + + +# --------------------------------------------------------------------------- +# TC-03 — 15-staging-log.md render: machine key + 52c schema + INFRA-WAIVED (FR-4 / AC-2) +# --------------------------------------------------------------------------- +def test_tc03_log_render_schema_and_infra_waived(): + stdout = "B6 OK\nINFRA-WAIVED: B9 image-build skipped (sandbox)\nDONE" + body = staging_runner.build_staging_log("ORCH-115", 0, "SUCCESS", stdout) + assert "staging_status: SUCCESS" in body # UPPERCASE machine key + for field in ("work_item: ORCH-115", "stage: deploy-staging", + "author_agent: staging-runner", "status: success", + "created_at:", "model_used: n/a"): + assert field in body, f"missing 52c field: {field}" + # INFRA-WAIVED line copied into the body for observability. + assert "INFRA-WAIVED: B9 image-build skipped (sandbox)" in body + + failed = staging_runner.build_staging_log("ORCH-115", 1, "FAILED", "boom") + assert "staging_status: FAILED" in failed + assert "status: failed" in failed + + +# --------------------------------------------------------------------------- +# TC-04 — generated log read by the UNCHANGED _parse_staging_status (AC-2) +# --------------------------------------------------------------------------- +def test_tc04_gate_parser_unchanged(): + from src.qg.checks import _parse_staging_status + ok, reason = _parse_staging_status(staging_runner.build_staging_log("ORCH-115", 0, "SUCCESS")) + assert ok is True and "SUCCESS" in reason + bad, reason2 = _parse_staging_status(staging_runner.build_staging_log("ORCH-115", 2, "FAILED")) + assert bad is False and "FAILED" in reason2 + + +# --------------------------------------------------------------------------- +# TC-05 — launch_job intercepts deployer on deploy-staging BEFORE _spawn (AC-1) +# --------------------------------------------------------------------------- +def test_tc05_launch_job_intercepts_before_spawn(monkeypatch): + tid = _make_task("deploy-staging") + jid = _make_job("deployer", "orchestrator", tid) + lr = AgentLauncher() + spawn = MagicMock(return_value=999) + monkeypatch.setattr(lr, "_spawn", spawn) + run_gate = MagicMock() + monkeypatch.setattr(staging_runner, "run_staging_gate", run_gate) + + ret = lr.launch_job(_job_dict(jid, "deployer", "orchestrator", tid)) + + assert ret is None # no agent_runs row + spawn.assert_not_called() # LLM path NOT taken + run_gate.assert_called_once() # deterministic runner ran + # jobs row driven to done by the launcher (no monitor/agent). + conn = get_db() + status = conn.execute("SELECT status FROM jobs WHERE id=?", (jid,)).fetchone()[0] + conn.close() + assert status == "done" + + +# --------------------------------------------------------------------------- +# TC-06 — stage discriminator: deployer on `deploy` is NOT intercepted (AC-1 / R-1) +# --------------------------------------------------------------------------- +def test_tc06_stage_discriminator_prod_not_intercepted(monkeypatch): + tid = _make_task("deploy") # prod edge, not deploy-staging + jid = _make_job("deployer", "orchestrator", tid) + lr = AgentLauncher() + spawn = MagicMock(return_value=42) + monkeypatch.setattr(lr, "_spawn", spawn) + run_gate = MagicMock() + monkeypatch.setattr(staging_runner, "run_staging_gate", run_gate) + + ret = lr.launch_job(_job_dict(jid, "deployer", "orchestrator", tid)) + + assert ret == 42 # _spawn path taken + spawn.assert_called_once() + run_gate.assert_not_called() + assert staging_runner.should_intercept(_job_dict(jid, "deployer", "orchestrator", tid)) is False + + +def test_tc06_non_self_repo_not_intercepted(monkeypatch): + tid = _make_task("deploy-staging", repo="enduro-trails", wi="ET-9", branch="feature/ET-9-x") + jid = _make_job("deployer", "enduro-trails", tid) + assert staging_runner.should_intercept(_job_dict(jid, "deployer", "enduro-trails", tid)) is False + + +# --------------------------------------------------------------------------- +# TC-07 — routing equivalence: SUCCESS -> advance; FAILED -> same path (AC-4) +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("rc,expected", [(0, "SUCCESS"), (1, "FAILED")]) +def test_tc07_advance_initiated_like_llm(monkeypatch, tmp_path, rc, expected): + tid = _make_task("deploy-staging") + monkeypatch.setattr(staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=rc, stdout="out", stderr="", timed_out=False)) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + advance.assert_called_once() + kwargs = advance.call_args.kwargs + assert kwargs["current_stage"] == "deploy-staging" + assert kwargs["finished_agent"] == "deployer" + assert kwargs["work_item_id"] == "ORCH-115" + # The log written for the gate carries the matching verdict. + assert f"staging_status: {expected}" in _read_log(tmp_path, "orchestrator", "feature/ORCH-115-x", "ORCH-115") + + +# --------------------------------------------------------------------------- +# TC-08 — kill-switch: enabled=False -> LLM path via _spawn (AC-6) +# --------------------------------------------------------------------------- +def test_tc08_killswitch_falls_back_to_spawn(monkeypatch): + monkeypatch.setattr(cfg.settings, "staging_runner_enabled", False) + tid = _make_task("deploy-staging") + jid = _make_job("deployer", "orchestrator", tid) + lr = AgentLauncher() + spawn = MagicMock(return_value=7) + monkeypatch.setattr(lr, "_spawn", spawn) + run_gate = MagicMock() + monkeypatch.setattr(staging_runner, "run_staging_gate", run_gate) + + ret = lr.launch_job(_job_dict(jid, "deployer", "orchestrator", tid)) + + assert ret == 7 + spawn.assert_called_once() # byte-for-byte the prior LLM-deployer path + run_gate.assert_not_called() + + +# --------------------------------------------------------------------------- +# TC-09 — anti-drift NFR-1: STAGE_TRANSITIONS / QG_CHECKS / schema untouched (AC-5) +# --------------------------------------------------------------------------- +def test_tc09_pipeline_contract_unchanged(): + from src.stages import STAGE_TRANSITIONS + from src.qg.checks import QG_CHECKS + # The deploy-staging edge + its gate are byte-for-byte the prior contract. + assert STAGE_TRANSITIONS["deploy-staging"] == { + "next": "deploy", "agent": "deployer", "qg": "check_staging_status" + } + assert "check_staging_status" in QG_CHECKS + # No new ORCH-115 table/column: only the existing tables exist. + conn = get_db() + tables = {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'").fetchall()} + conn.close() + assert not any("staging_runner" in t for t in tables) + # The machine key is not renamed: the runner emits `staging_status:`. + assert "staging_status:" in staging_runner.build_staging_log("ORCH-115", 0, "SUCCESS") + + +# --------------------------------------------------------------------------- +# TC-10 — never-raise / fail-safe: tool-error never a silent advance/false green (AC-7) +# --------------------------------------------------------------------------- +def test_tc10_nonzero_exit_is_failed_and_advances(monkeypatch, tmp_path): + tid = _make_task("deploy-staging") + monkeypatch.setattr(staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=3, stdout="fail", stderr="", timed_out=False)) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + advance.assert_called_once() + assert "staging_status: FAILED" in _read_log(tmp_path, "orchestrator", "feature/ORCH-115-x", "ORCH-115") + + +def test_tc10_timeout_defers_without_advance(monkeypatch): + tid = _make_task("deploy-staging") + monkeypatch.setattr(cfg.settings, "staging_runner_infra_max_retries", 2) + monkeypatch.setattr(staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=None, stdout="", stderr="", timed_out=True)) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + advance.assert_not_called() # NO silent advance / false green on infra hiccup + # A fresh deployer job was re-queued with the restart-safe marker (bounded DEFER). + conn = get_db() + n = conn.execute( + "SELECT COUNT(*) FROM jobs WHERE task_id=? AND task_content LIKE ?", + (tid, f"%{staging_runner._INFRA_RETRY_MARKER}%"), + ).fetchone()[0] + conn.close() + assert n == 1 + assert staging_runner._STAGING_RUNNER_COUNTERS["deferred"] == 1 + assert staging_runner._STAGING_RUNNER_COUNTERS["tool_error"] == 1 + + +def test_tc10_tool_error_budget_exhausted_fails_closed(monkeypatch, tmp_path): + tid = _make_task("deploy-staging") + monkeypatch.setattr(cfg.settings, "staging_runner_infra_max_retries", 0) # exhausted immediately + monkeypatch.setattr(staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=None, stdout="", stderr="", timed_out=True)) + advance = MagicMock() + monkeypatch.setattr(stage_engine, "advance_stage", advance) + + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + advance.assert_called_once() # fail-closed: write FAILED + advance (existing rollback) + assert "staging_status: FAILED" in _read_log(tmp_path, "orchestrator", "feature/ORCH-115-x", "ORCH-115") + + +def test_tc10_run_gate_never_raises(monkeypatch): + tid = _make_task("deploy-staging") + + def boom(): + raise RuntimeError("docker exec exploded") + monkeypatch.setattr(staging_runner, "run_staging_suite", boom) + # Must NOT raise (AC-7): the worker is protected even on an unexpected error. + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + + +def test_tc10_launcher_contains_runner_fault(monkeypatch): + tid = _make_task("deploy-staging") + jid = _make_job("deployer", "orchestrator", tid) + lr = AgentLauncher() + monkeypatch.setattr(lr, "_spawn", MagicMock()) + monkeypatch.setattr(staging_runner, "run_staging_gate", + MagicMock(side_effect=RuntimeError("kaboom"))) + # The launcher wrapper contains the fault -> job marked failed, never raises. + ret = lr.launch_job(_job_dict(jid, "deployer", "orchestrator", tid)) + assert ret is None + conn = get_db() + status = conn.execute("SELECT status FROM jobs WHERE id=?", (jid,)).fetchone()[0] + conn.close() + assert status == "failed" + + +# --------------------------------------------------------------------------- +# TC-11 — timeout resolution + propagation (NFR-4 / AC-9) +# --------------------------------------------------------------------------- +def test_tc11_resolve_timeout_default_on_bad_value(monkeypatch): + monkeypatch.setattr(cfg.settings, "staging_runner_timeout_s", 1234) + assert staging_runner._resolve_timeout() == 1234 + for bad in (0, -5, "abc", None): + monkeypatch.setattr(cfg.settings, "staging_runner_timeout_s", bad) + assert staging_runner._resolve_timeout() == staging_runner._DEFAULT_TIMEOUT_S + + +def test_tc11_timeout_passed_to_proc_group(monkeypatch): + monkeypatch.setattr(cfg.settings, "staging_runner_timeout_s", 321) + monkeypatch.setattr(cfg.settings, "subprocess_tree_kill_enabled", True) + captured = {} + + def fake_run(cmd, *, cwd, timeout, grace_s, tree_kill): + captured.update(cmd=cmd, timeout=timeout, tree_kill=tree_kill) + return ProcResult(returncode=0, stdout="", stderr="", timed_out=False) + monkeypatch.setattr(staging_runner.proc_group, "run_in_process_group", fake_run) + + staging_runner.run_staging_suite() + assert captured["timeout"] == 321 + assert captured["tree_kill"] is True + assert captured["cmd"][:2] == ["docker", "exec"] + + +# --------------------------------------------------------------------------- +# TC-12 — self-hosting safety: no forbidden literals in the runner command (AC-8) +# --------------------------------------------------------------------------- +def test_tc12_command_has_no_dangerous_literals(): + cmd = " ".join(staging_runner.build_staging_command()) + forbidden = ("compose", "up -d", "--build", "8500", "force", "push", ".env", + "rm ", "restart") + for token in forbidden: + assert token not in cmd, f"forbidden literal {token!r} in runner command: {cmd}" + # It IS the staging suite against 8501 via docker exec. + assert "docker exec" in cmd + assert "staging_check.py" in cmd + assert "8501" in cmd + assert "--mode stub" in cmd + + +# --------------------------------------------------------------------------- +# TC-13 — observability: /queue block + structured verdict log (AC-10) +# --------------------------------------------------------------------------- +def test_tc13_snapshot_shape(): + snap = staging_runner.snapshot() + for k in ("enabled", "repos", "timeout_s", "infra_max_retries", + "runs", "success", "failed", "tool_error", "deferred"): + assert k in snap, f"snapshot missing key {k}" + + +def test_tc13_queue_endpoint_includes_block(monkeypatch): + import asyncio + from src import main + payload = asyncio.run(main.queue()) + assert "staging_runner" in payload + assert "enabled" in payload["staging_runner"] + + +def test_tc13_structured_verdict_log_distinguishes_outcomes(monkeypatch, caplog, tmp_path): + tid = _make_task("deploy-staging") + monkeypatch.setattr(stage_engine, "advance_stage", MagicMock()) + # code-pass + monkeypatch.setattr(staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=0, stdout="", stderr="", timed_out=False)) + with caplog.at_level("INFO", logger="orchestrator.staging_runner"): + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + assert any("outcome=code-pass" in r.message for r in caplog.records) + + caplog.clear() + # tool-error + monkeypatch.setattr(cfg.settings, "staging_runner_infra_max_retries", 2) + monkeypatch.setattr(staging_runner, "run_staging_suite", + lambda: ProcResult(returncode=None, stdout="", stderr="", timed_out=True)) + with caplog.at_level("WARNING", logger="orchestrator.staging_runner"): + staging_runner.run_staging_gate(_job_dict(1, "deployer", "orchestrator", tid)) + assert any("outcome=tool-error" in r.message for r in caplog.records) + + +def test_tc13_snapshot_never_raises(monkeypatch): + class Boom: + def __getattr__(self, name): + raise RuntimeError("boom") + monkeypatch.setattr(staging_runner, "settings", Boom()) + snap = staging_runner.snapshot() + assert snap["enabled"] is False