From eadfd8419b575e6511a662d0142c7a1aa783c9e1 Mon Sep 17 00:00:00 2001 From: claude-bot Date: Wed, 10 Jun 2026 01:04:21 +0300 Subject: [PATCH] feat(coverage): deterministic test-coverage gate on deploy-staging->deploy edge (ORCH-027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a deterministic (no-LLM) coverage sub-gate that blocks coverage degradation before a task branch merges into `main`. Existing gates judge only by the FACT of passing (check_ci_green / check_tests_passed / merge-gate re-test), not by completeness — so a batch autonomous run (ORCH-088) silently erodes coverage. Pattern mirrors the security-gate (ORCH-022): leaf src/coverage_gate.py (never-raise) + thin check_coverage_gate in QG_CHECKS + _handle_coverage_gate splice in advance_stage, run AFTER merge-gate (measured on the caught-up HEAD that lands in main) and BEFORE image-freshness (fail before the expensive docker rebuild). - measure_coverage: pytest --cov=src --cov-report=json in the per-branch worktree -> line coverage %; None on tool error -> fail-open + WARNING by default (FR-6). - compute_coverage_verdict (pure): absolute | baseline | both + epsilon (NFR-4 anti-flap); baseline None -> bootstrap (absolute-only). - coverage_baseline DB table (additive, CREATE TABLE IF NOT EXISTS) + ratchet-up in _handle_merge_verify (deploy->done): atomic compare-and-set under merge-lease, never decreases; bootstrap on first merge. - Artefact 18-coverage-report.md (coverage_status: frontmatter, single source of truth); GET /queue `coverage` block; FAIL -> Telegram; optional POST /coverage/baseline override. - Flags ORCH_COVERAGE_* (kill-switch + self-hosting-only scope) -> enduro untouched; STAGE_TRANSITIONS / existing check_* / verdict keys byte-for-byte unchanged (NFR-5/AC-8). - pytest-cov==5.0.0 added to requirements.txt. Tests: tests/test_coverage_gate.py (TC-01..TC-15). Frozen QG-registry anti-regress tests + deploy-staging edge tests updated for the new sub-gate. Full suite green. Docs: README / adr-0029 / PIPELINE_DOCS / 18-coverage-report.md template (architecture stage) + CHANGELOG / CLAUDE.md / .env.example (this PR). Refs: ORCH-027 Co-Authored-By: Claude Opus 4.8 --- .env.example | 22 ++ CHANGELOG.md | 12 +- CLAUDE.md | 47 ++- requirements.txt | 4 + src/config.py | 32 ++ src/coverage_gate.py | 616 +++++++++++++++++++++++++++++ src/db.py | 128 ++++++ src/main.py | 24 ++ src/qg/checks.py | 18 + src/stage_engine.py | 108 +++++ tests/test_config.py | 1 + tests/test_coverage_gate.py | 471 ++++++++++++++++++++++ tests/test_plane_status_model.py | 1 + tests/test_qg_registry_snapshot.py | 1 + tests/test_stage_engine.py | 12 + tests/test_stages_invariants.py | 1 + 16 files changed, 1496 insertions(+), 2 deletions(-) create mode 100644 src/coverage_gate.py create mode 100644 tests/test_coverage_gate.py diff --git a/.env.example b/.env.example index 90c59c4..b422fbd 100644 --- a/.env.example +++ b/.env.example @@ -372,6 +372,28 @@ ORCH_SECURITY_SCAN_TIMEOUT_S=300 ORCH_SECURITY_DEP_AUDIT_FAIL_CLOSED=false ORCH_SECURITY_SECRETS_BLOCK=true +# ORCH-027: coverage-gate (deterministic test-coverage) on the deploy-staging -> +# deploy edge, run AFTER the merge-gate and BEFORE image-freshness. Measures line +# coverage of src/ with pytest-cov in the per-branch worktree, compares to an absolute +# floor and/or the ratchet baseline of `main`; FAIL -> rollback to development + +# developer-retry (cap 3). Verdict in the 18-coverage-report.md frontmatter +# (coverage_status:). See ADR-001-coverage-gate.md. +# GATE_ENABLED -> global kill-switch; false -> pipeline 1:1 as before ORCH-027. +# GATE_REPOS -> CSV of repos where the gate is REAL; empty -> only self-hosting. +# MIN_PERCENT -> absolute floor (% line coverage) for policy absolute/both. +# POLICY -> absolute | baseline | both (default both). +# EPSILON -> noise tolerance (%) at the boundary (anti-flap). +# TOOL_FAIL_CLOSED -> strict mode: a coverage-tool error -> FAIL instead of the +# default fail-open + warning (anti-loop). Default false. +# RUN_TIMEOUT_S -> wall-clock budget for the pytest --cov run. +ORCH_COVERAGE_GATE_ENABLED=true +ORCH_COVERAGE_GATE_REPOS= +ORCH_COVERAGE_MIN_PERCENT=0.0 +ORCH_COVERAGE_POLICY=both +ORCH_COVERAGE_EPSILON=0.5 +ORCH_COVERAGE_TOOL_FAIL_CLOSED=false +ORCH_COVERAGE_RUN_TIMEOUT_S=900 + # ORCH-021: post-deploy production monitoring + degradation reaction. After the # terminal deploy->done transition for an applicable repo, a reserved-agent job # `post-deploy-monitor` (no LLM, modelled on deploy-finalizer) probes prod over a diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ae9791..eee6b9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,17 @@ Формат: [Keep a Changelog](https://keepachangelog.com/). Записи — на смысловой PR/задачу. ## [Unreleased] -- **Live-карточка трекера: HTML-инъекция «<1м» больше не застывает карточку — экранирование всех данных-полей на границе рендера** (ORCH-095, `fix`): карточка задачи (`src/notifications.py::render_task_tracker`) шлётся/редактируется с `parse_mode=HTML`. `_fmt_minutes` для стадии < 60 с возвращает литерал `"<1м"`, который интерполировался в HTML-текст **сырым** → Telegram парсит `<1м` как открывающий тег → `editMessageText` отвечает `400 can't parse entities: Unsupported start tag "1м"` → `edit_telegram` классифицирует как `EDIT_FAILED` → `update_task_tracker` делает ранний `return` (анти-дубль ORCH-087) → **карточка застывает** (детерминированно воспроизведено 09.06 на ORCH-093, `message_id 18854`). Корневой класс шире одного `<1м`: все подставляемые **данные** (длительности, статус-лейбл, модель, эффорт, токены/стоимость) вставлялись сырыми; экранирован был только заголовок (`esc_title`) и href/label внутри `plane_issue_link`. **Аддитивно, never-raise, без нового поведения конвейера:** `STAGE_TRANSITIONS` / `QG_CHECKS` / `check_*` / транспорт нотификаций / схема БД — **не тронуты** (затронут ровно один модуль индикативного слоя); kill-switch не требуется (исправление дефекта корректности, откат = `git revert`). +- **Детерминированный гейт покрытия тестами — защита от тихой деградации coverage перед merge в `main`** (ORCH-027, `feat`): существующие тестовые гейты (`check_ci_green`, `check_tests_passed`, merge-gate re-test) судят только по **факту** прохождения, не по **полноте** — ни один не замечает «300 строк кода, 0 тестов», и при пакетном автономном прогоне (ORCH-088) покрытие монотонно деградирует. Введён детерминированный (без LLM) под-гейт ребра `deploy-staging → deploy` по образцу security-гейта (ORCH-022): leaf `src/coverage_gate.py` (never-raise) + тонкая обёртка `check_coverage_gate` в `QG_CHECKS` + врезка `_handle_coverage_gate` в `advance_stage`. **Аддитивно:** `STAGE_TRANSITIONS` / семантика существующих `check_*` / machine-verdict ключи (`verdict:`/`result:`/`deploy_status:`/`staging_status:`/`security_status:`) — байт-в-байт прежние; новая БД-таблица аддитивна (NFR-5/AC-8). См. `docs/work-items/ORCH-027/06-adr/ADR-001-coverage-gate.md`, сквозной `docs/architecture/adr/adr-0029-coverage-gate.md`. + - **Точка/порядок (D1, AC-2):** под-гейт исполняется **ПОСЛЕ merge-gate** (покрытие меряется на догнанном `auto_rebase_onto_main` HEAD — ровно том коде, что landed в `main`) и **ДО image-freshness** (фейл до дорогого docker-rebuild). FAIL → штатный откат на `development` (+ инкремент developer-retry, cap `MAX_DEVELOPER_RETRIES`) **и освобождение merge-lease** (merge-gate держал его на своём PASS — зеркало image-freshness rollback, TR-2). `STAGE_TRANSITIONS` не меняется (под-гейт, как security/merge/image-freshness). + - **Измерение (D2, FR-1/AC-1):** `python -m pytest tests/ --cov=src --cov-report=json` в изолированном per-branch worktree (`ensure_worktree`, прецедент `check_tests_local`); метрика — `totals.percent_covered` (line coverage `src/`). Измеритель инкапсулирован за `measure_coverage(repo, branch) -> float | None` (стек-расширяемость BR-6: jest/jacoco — новая ветка `measure_*`, без переписывания ядра). Тайм-аут `coverage_run_timeout_s`. Новая pip-зависимость `pytest-cov==5.0.0` (offline на момент замера). + - **Чистая функция решения (D3, FR-2/AC-3):** `compute_coverage_verdict(measured, baseline, floor, policy, epsilon) -> (ok, reason)` — детерминированная, без LLM/IO. `absolute` → `measured ≥ floor−ε`; `baseline` → `measured ≥ baseline−ε`; `both` (дефолт) → оба; `baseline is None` (bootstrap) → baseline-условие не применяется (нельзя регрессировать против пустоты). `epsilon` — допуск на шум измерения (NFR-4, анти-флап у границы). Покрыто unit-тестами всех режимов/границ/epsilon. + - **Базовая линия + ratchet (D4/D5, FR-4/AC-4):** аддитивная БД-таблица `coverage_baseline(repo PK, coverage, source_sha, updated_at)` (`CREATE TABLE IF NOT EXISTS`, паттерн `repo_freeze`/`job_deps`; существующие таблицы не мигрируются). Хелперы `db.get_coverage_baseline`/`ratchet_coverage_baseline`/`set_coverage_baseline`/`all_coverage_baselines`. Наращивание **только вверх** в choke-point подтверждённого merge `_handle_merge_verify` (ребро `deploy → done`): `coverage_gate.ratchet_baseline_on_merge` читает измеренное из `18-coverage-report.md` (single source of truth) и применяет **атомарный compare-and-set** `UPDATE … WHERE coverage <= measured` (или `INSERT` — bootstrap) под держимым merge-lease (ORCH-043) → базовая линия никогда не падает даже при гонке. Меньшее значение базовую линию не понижает. + - **Условность + fail-open (D6, FR-5/FR-6/AC-5/AC-6):** `coverage_gate_applies(repo)` (локально) ПЕРВЫМ — дорогой прогон только при `applies==True`. `coverage_gate_enabled=False` → инертно (1:1 как до ORCH-027); `coverage_gate_repos` (CSV; **пусто → self-hosting only** `is_self_hosting_repo`, как security/merge/image-freshness) → enduro-trails не затронут (no-op `(True, "N/A")`). Ошибка/недоступность coverage-инструмента или непарсимая метрика → **fail-open + WARNING** по умолчанию (`coverage_tool_fail_closed=False`, анти-петля по образцу ORCH-061/022 dep-audit); флаг переключает в fail-closed. + - **Машинный вердикт + наблюдаемость (D7/D8, FR-7/AC-9):** артефакт `18-coverage-report.md` (frontmatter `coverage_status: PASS|FAIL` + `measured_coverage`/`baseline`/`floor`/`policy`/`epsilon`/`delta`), вердикт читается ТОЛЬКО из frontmatter через `src/frontmatter.parse_frontmatter` (ORCH-052c, регистр фиксирован); гейт сам пишет отчёт и читает вердикт обратно из того же файла (single source of truth, как `security_status:`). Read-only блок `coverage` в `GET /queue` (kill-switch/scope/policy/floor/epsilon/per-repo baselines). При FAIL — `send_telegram` с кликабельным номером (`link_for`), измеренным покрытием, порогом/базовой линией и дельтой. Опциональный ручной override `POST /coverage/baseline?repo=…&value=…` (по образцу `POST /serial-gate/unfreeze`) для легитимного разового снижения покрытия. + - **Self-hosting безопасность (NFR-1/NFR-3/AC-7):** leaf не импортирует `stage_engine`; любое исключение перехвачено (never-raise); гейт только мерит/читает/пишет/решает — не деплоит, не рестартит прод-контейнер, не пушит/форс-пушит `main` (структурно проверено AST-тестом TC-12). Прод-деплой ORCH-027 — строго через staging-гейт (8501), без рестарта прод-контейнера (лейбл `arch:major-change`). + - **Флаги (`config.py`, env `ORCH_COVERAGE_*`, `.env.example`):** `coverage_gate_enabled` (kill-switch), `coverage_gate_repos`, `coverage_min_percent` (дефолт 0.0 — безопасный раскат: no-regression ведёт ratchet-базовая линия, floor не фейлит в день один), `coverage_policy` (дефолт `both`), `coverage_epsilon` (0.5), `coverage_tool_fail_closed` (False), `coverage_run_timeout_s` (900). Откат: `ORCH_COVERAGE_GATE_ENABLED=false` → полный no-op (мгновенный обратимый kill-switch). + - **Инфра-предусловие:** добавить `pytest-cov` в прод/staging-образ (`requirements.txt`). При первом применимом merge базовая линия засевается фактическим покрытием `main` (bootstrap). Тесты: `tests/test_coverage_gate.py` (TC-01…TC-15: режимы/границы/epsilon verdict, ratchet up-only + bootstrap + per-repo изоляция, applies/kill-switch, fail-open/closed, never-raise, write/read-back отчёта, self-hosting AST-safety, интеграция в `advance_stage` с откатом+release lease, реальное измерение pytest-cov на фикстур-репо + тайм-аут, snapshot + неизменность `QG_CHECKS`/`STAGE_TRANSITIONS`). Обновлены анти-регресс-реестры `QG_CHECKS` (`test_config`/`test_plane_status_model`/`test_qg_registry_snapshot`/`test_stages_invariants`) и edge-тесты `test_stage_engine` (`check_coverage_gate: _pass`). Полный регресс `tests/ -q` зелёный. +- **Live-карточка трекера: HTML-инъекция «<1м» больше не застывает карточку — экранирование всех данных-полей на границе рендера** (ORCH-095, `fix`): карточка задачи (`src/notifications.py::render_task_tracker`) шлётся/редактируется с `parse_mode=HTML`. `_fmt_minutes` для стадии < 60 с возвращает литерал `"<1м"`, который интерполировался в HTML-текст **сырым** → Telegram парсит `<1м` как открывающий тег → `editMessageText` отвечает `400 can't parse entities: Unsupported start tag "1м"` → `edit_telegram` классифицирует как `EDIT_FAILED` → `update_task_tracker` делает ранний `return` (анти-дубль ORCH-087) → **карточка застывает** (детерминированно воспроизведено 09.06 на ORCH-093, `message_id 18854`). Корневой класс шире одного `<1м`: все подставляемые **данные** (длительности, статус-лейбл, модель, эффорт, токены/стоимость) вставлялись сырыми; экранирован был только заголовок (`esc_title`) и href/label внутри `plane_issue_link`. **Аддитивно, never-raise, без нового поведения конвейера:** `STAGE_TRANSITIONS` / `QG_CHECKS` / `check_*` / транспорт нотификаций / схема БД — **не тронуты** (затронут ровно один модуль индикативного слоя); kill-switch не требуется (исправление дефекта корректности, откат = `git revert`). карточка задачи (`src/notifications.py::render_task_tracker`) шлётся/редактируется с `parse_mode=HTML`. `_fmt_minutes` для стадии < 60 с возвращает литерал `"<1м"`, который интерполировался в HTML-текст **сырым** → Telegram парсит `<1м` как открывающий тег → `editMessageText` отвечает `400 can't parse entities: Unsupported start tag "1м"` → `edit_telegram` классифицирует как `EDIT_FAILED` → `update_task_tracker` делает ранний `return` (анти-дубль ORCH-087) → **карточка застывает** (детерминированно воспроизведено 09.06 на ORCH-093, `message_id 18854`). Корневой класс шире одного `<1м`: все подставляемые **данные** (длительности, статус-лейбл, модель, эффорт, токены/стоимость) вставлялись сырыми; экранирован был только заголовок (`esc_title`) и href/label внутри `plane_issue_link`. **Аддитивно, never-raise, без нового поведения конвейера:** `STAGE_TRANSITIONS` / `QG_CHECKS` / `check_*` / транспорт нотификаций / схема БД — **не тронуты** (затронут ровно один модуль индикативного слоя); kill-switch не требуется (исправление дефекта корректности, откат = `git revert`). - **Экранирование на границе рендера, не в источнике (ADR-001 D1/D2, AC-1/AC-2):** новый модуль-локальный хелпер `_esc(x) = html.escape(str(x))` (never-raise → `""` на исключении) оборачивает каждое подставляемое **данные-значение** (категория D) ровно один раз в точке интерполяции в `render_task_tracker`/`_stage_line`: длительности (`_fmt_minutes`/`_capped_review_str`), статус-лейбл (`_card_status_label`), модель (`short_model_name`), эффорт (`_run_effort`), токены/стоимость (`fmt_tokens`/`fmt_cost`). Функции-источники остаются **HTML-агностичными** (данные, не разметка): `src/usage.py` и `_fmt_minutes` не тронуты — `_fmt_minutes` продолжает возвращать `"<1м"`, безопасность даёт escape на границе (`<1м` рендерится оператору визуально идентично `<1м` → видимый формат не меняется). - **Категория M (намеренная разметка) неприкосновенна (D5, AC-3):** кликабельный номер задачи `num_html` (`plane_issue_link`, внутри уже экранированы href+label), `link_for(...)` в строке «⏳ ждёт …», `_done_link(...)` («🔗 PR #n · 📦 Внедрено») и уже-экранированный `esc_title` через `_esc` **не** проходят → остаются валидным HTML, номер остаётся кликабельным. Двойное экранирование (`&lt;`) структурно исключено: D-слот → `_esc` ровно один раз, M-слот → as-is. - **Defence-in-depth (D3):** экранируются и сейчас-безопасные D-поля (токены/стоимость/модель дают только цифры/`.`/`k`/`M`/`$`/`^claude-…$`) — escape для них no-op, выгода — структурный инвариант «каждый D-слот экранирован», устойчивый к будущей смене формата источника. diff --git a/CLAUDE.md b/CLAUDE.md index 0035cb0..6181f95 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -153,6 +153,51 @@ created → analysis → architecture → development → review → testing → `docs/work-items/ORCH-090/06-adr/ADR-001-stop-cancel-task.md`, `docs/architecture/adr/adr-0026-stop-cancel-task.md`. +## Гейт покрытия тестами (ORCH-027) +Существующие тестовые гейты (`check_ci_green`, `check_tests_passed`, merge-gate re-test) судят +только по **факту** прохождения, не по **полноте** — ни один не замечает «300 строк кода, 0 +тестов», и при пакетном автономном прогоне (ORCH-088) покрытие монотонно деградирует. Введён +**детерминированный (без LLM) под-гейт ребра `deploy-staging → deploy`** по образцу security-гейта +(ORCH-022): leaf `src/coverage_gate.py` (never-raise) + тонкая обёртка `check_coverage_gate` в +`QG_CHECKS` + врезка `_handle_coverage_gate` в `advance_stage`. **Инвариант:** `STAGE_TRANSITIONS` / +семантика существующих `check_*` / machine-verdict ключи (`verdict:`/`result:`/`deploy_status:`/ +`staging_status:`/`security_status:`) — байт-в-байт прежние; новая БД-таблица аддитивна (NFR-5). +- **Точка/порядок:** **ПОСЛЕ merge-gate** (покрытие меряется на догнанном `auto_rebase_onto_main` + HEAD — ровно том коде, что landed в `main`) и **ДО image-freshness** (фейл до дорогого + docker-rebuild). Порядок под-гейтов: **security → merge → coverage → image-freshness.** FAIL → + штатный откат на `development` (+ инкремент developer-retry, cap `MAX_DEVELOPER_RETRIES`) **и + освобождение merge-lease** (merge-gate держал его на своём PASS — зеркало image-freshness rollback). +- **Измерение:** `python -m pytest tests/ --cov=src --cov-report=json` в изолированном per-branch + worktree (`ensure_worktree`); метрика — `totals.percent_covered` (line coverage `src/`). Измеритель + за `measure_coverage(repo, branch) -> float | None` (стек-расширяемость BR-6). Тайм-аут + `coverage_run_timeout_s`. Новая pip-зависимость `pytest-cov`. +- **Решение — чистая функция** `compute_coverage_verdict(measured, baseline, floor, policy, epsilon) + -> (ok, reason)`: `absolute` → `measured ≥ floor−ε`; `baseline` → `measured ≥ baseline−ε`; `both` + (дефолт) → оба; `baseline is None` (bootstrap) → baseline-условие не применяется. `epsilon` — + допуск на шум измерения (анти-флап у границы). +- **Базовая линия — аддитивная БД-таблица** `coverage_baseline(repo PK, coverage, source_sha, + updated_at)` (`CREATE TABLE IF NOT EXISTS`; хелперы `db.get_coverage_baseline`/ + `ratchet_coverage_baseline`/`set_coverage_baseline`). Наращивание **только вверх** в choke-point + подтверждённого merge `_handle_merge_verify` (ребро `deploy → done`): `ratchet_baseline_on_merge` + читает измеренное из `18-coverage-report.md` (single source of truth), атомарный compare-and-set + `UPDATE … WHERE coverage <= measured` под держимым merge-lease (ORCH-043) → базовая линия не падает + даже при гонке; bootstrap засевается первым применимым merge. +- **Условность (как ORCH-22/43/58):** `coverage_gate_enabled` (kill-switch; `False` → 1:1 как до + ORCH-027) + `coverage_gate_repos` (CSV; **пусто → self-hosting only** `is_self_hosting_repo` → + enduro не затронут, no-op `(True, "N/A")`); `applies(repo)` (локально) ПЕРВЫМ — дорогой прогон + только при `applies==True`. Ошибка инструмента/непарсимая метрика → **fail-open + WARNING** по + умолчанию (`coverage_tool_fail_closed=False`, анти-петля); флаг → fail-closed. +- **Артефакт `18-coverage-report.md`** (frontmatter `coverage_status: PASS|FAIL` + + `measured_coverage`/`baseline`/`floor`/`policy`/`epsilon`/`delta`), вердикт читается ТОЛЬКО из + frontmatter через `src/frontmatter.py` (single source of truth, как `security_status:`). + Наблюдаемость — read-only блок `coverage` в `GET /queue`; при FAIL — `send_telegram` с кликабельным + номером, измеренным/порогом/дельтой; опциональный ручной override `POST /coverage/baseline`. + Флаги `ORCH_COVERAGE_*` (`MIN_PERCENT`/`POLICY`/`EPSILON`/`TOOL_FAIL_CLOSED`/`RUN_TIMEOUT_S`). + Self-hosting-безопасно: гейт только мерит/читает/пишет/решает — не деплоит/не рестартит прод/не + пушит `main`. **Инфра-предусловие:** `pytest-cov` в прод/staging-образе. Детали — + `docs/work-items/ORCH-027/06-adr/ADR-001-coverage-gate.md`, + `docs/architecture/adr/adr-0029-coverage-gate.md`. + ## Конвенции - Conventional Commits (`feat:`, `fix:`, `docs:`, `refactor:`, `test:`) - Ветки: `feature/ORCH-NNN-slug`, `fix/ORCH-NNN-slug` @@ -162,7 +207,7 @@ created → analysis → architecture → development → review → testing → - Машинные вердикты Quality Gate — строго YAML-frontmatter (`verdict:`, `deploy_status:`, `staging_status:`, `security_status:`), никогда проза. **ORCH-52c (ORCH-076):** парсинг frontmatter сведён к единому контракту `src/frontmatter.py` (reader `read_frontmatter_value` — BC; единый парс-примитив `parse_frontmatter`; writer `render/write_frontmatter`; валидатор схемы `validate_schema`/`REQUIRED_FIELDS` — warning-only по умолчанию, hard-fail только под kill-switch `frontmatter_validation_strict`, дефолт `False`). Пять вердикт-парсеров (`check_reviewer_verdict`, `_parse_tests_verdict`, `_parse_deploy_status`, `_parse_staging_status`, `parse_security_status`) читают через ОДНУ точку парсинга; семантика вердиктов и `STAGE_TRANSITIONS`/состав `QG_CHECKS` — 1:1. Формальная спека «стадия → обязательный выход» + обязательная frontmatter-схема — `docs/_standards/HANDOFF_PROTOCOL.md` ## Артефакты задачи (`docs/work-items//`) -`00-business-request.md`, `01-brd.md`, `02-trz.md`, `03-acceptance-criteria.md`, `04-test-plan.yaml`, `06-adr/ADR-NNN-slug.md`, `07-infra-requirements.md`, `08-data-requirements.md`, `10-tech-risks.md`, `12-review.md`, `13-test-report.md`, `14-deploy-log.md`, `15-staging-log.md`, `16-post-deploy-log.md` (post-deploy наблюдение, ORCH-021), `17-security-report.md` (security-гейт: `security_status:`/secrets/deps, ORCH-022). +`00-business-request.md`, `01-brd.md`, `02-trz.md`, `03-acceptance-criteria.md`, `04-test-plan.yaml`, `06-adr/ADR-NNN-slug.md`, `07-infra-requirements.md`, `08-data-requirements.md`, `10-tech-risks.md`, `12-review.md`, `13-test-report.md`, `14-deploy-log.md`, `15-staging-log.md`, `16-post-deploy-log.md` (post-deploy наблюдение, ORCH-021), `17-security-report.md` (security-гейт: `security_status:`/secrets/deps, ORCH-022), `18-coverage-report.md` (coverage-гейт: `coverage_status:`/measured/baseline, ORCH-027). **Стандарт документов (ORCH-075, ORCH-52b):** структура каждого дока, карта «стадия→агент→документ→гейт→machine-key» и конвенция ADR-naming зафиксированы в `docs/_standards/PIPELINE_DOCS.md` (golden source); копируемые скелеты — в `docs/_templates/`. Перед написанием номерного дока бери скелет из `docs/_templates/` и не меняй имя machine-key frontmatter (регистр чувствителен — иначе гейт упадёт ложно). diff --git a/requirements.txt b/requirements.txt index 9aed60e..38a61c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,10 @@ pydantic-settings==2.5.0 httpx==0.27.0 pytest==8.3.3 pytest-asyncio==0.23.8 +# ORCH-027: coverage measurement for the coverage-gate. pytest-cov wraps coverage.py; +# the gate runs `pytest --cov=src --cov-report=json` in the per-branch worktree and +# reads totals.percent_covered (line coverage). Offline — no network at measure time. +pytest-cov==5.0.0 # ORCH-022: dependency audit (OSV/PyPI advisory) for the security-gate. Needs the # network at scan time -> an unreachable feed degrades fail-open + warning by # default (ADR-001 Р-3 / 07-infra I-2). gitleaks (secret-scan) is a pinned Go diff --git a/src/config.py b/src/config.py index dc48aa9..ec222e4 100644 --- a/src/config.py +++ b/src/config.py @@ -259,6 +259,38 @@ class Settings(BaseSettings): security_dep_audit_fail_closed: bool = False security_secrets_block: bool = True + # ORCH-027: deterministic test-coverage gate on the deploy-staging -> deploy edge + # (AFTER the merge-gate, BEFORE image-freshness). Measures line coverage of src/ + # under pytest-cov in the per-branch worktree, compares to an absolute floor and/or + # the ratchet baseline of `main`, and FAILs (rollback to development + developer + # retry) on degradation. Leaf src/coverage_gate.py (never-raise); machine verdict in + # 18-coverage-report.md frontmatter (coverage_status:). See ADR-001-coverage-gate.md. + # coverage_gate_enabled -> SINGLE kill-switch; False -> pipeline 1:1 as before + # ORCH-027 for everyone. Env ORCH_COVERAGE_GATE_ENABLED. + # coverage_gate_repos -> CSV of repos where the gate is REAL; empty -> only + # the self-hosting repo (orchestrator). Mirrors + # security_gate_repos / image_freshness_repos. + # coverage_min_percent -> absolute floor (% line coverage) for policy + # absolute/both. Default 0.0 -> safe rollout: the + # ratchet baseline drives no-regression, the floor + # never false-fails day one. + # coverage_policy -> absolute | baseline | both (default both): which + # condition(s) must hold (D3). + # coverage_epsilon -> small non-negative noise tolerance (%) so jitter at + # the boundary does not bounce a task (NFR-4). + # coverage_tool_fail_closed -> strict mode: a coverage-tool error -> FAIL instead + # of the default fail-open + warning (FR-6). Default + # False (anti-loop, precedent ORCH-061/022). + # coverage_run_timeout_s -> wall-clock budget for the pytest --cov run (mirrors + # merge_retest_timeout_s / security_scan_timeout_s). + coverage_gate_enabled: bool = True + coverage_gate_repos: str = "" + coverage_min_percent: float = 0.0 + coverage_policy: str = "both" + coverage_epsilon: float = 0.5 + coverage_tool_fail_closed: bool = False + coverage_run_timeout_s: int = 900 + # ORCH-061: tolerate KNOWN sandbox-infra FAILs (C9a/C9b) in the staging suite. # The self-hosting deploy-staging stage looped because scripts/staging_check.py # exited non-zero on ANY failed check, so two infra-only failures (sandbox bot diff --git a/src/coverage_gate.py b/src/coverage_gate.py new file mode 100644 index 0000000..d13c9f8 --- /dev/null +++ b/src/coverage_gate.py @@ -0,0 +1,616 @@ +"""Coverage-gate core (ORCH-027): deterministic test-coverage gate before merge. + +Background +---------- +The orchestrator runs autonomous development: the ``developer`` agent writes code +with no human filter, and on ``testing`` the ``tester`` agent decides for itself +whether the tests are enough. The existing test gates judge only by the FACT of +passing, never by COMPLETENESS: ``check_ci_green`` and ``check_tests_passed`` and +the merge-gate re-test all look at a pytest exit code. None of them notices "300 +lines of new code, 0 tests". Across a batch autonomous run (ORCH-088) that means a +monotonic erosion of coverage — every task shaves a corner on tests and the project +silently loses testability. + +This module provides the deterministic (no-LLM) primitives that the quality-gate +``check_coverage_gate`` (src/qg/checks.py) composes on the ``deploy-staging -> +deploy`` edge — run **AFTER the merge-gate** (so coverage is measured on the +caught-up HEAD that actually lands in ``main``) and **BEFORE image-freshness** (fail +before the expensive docker rebuild), mirroring the security-gate (ORCH-022): + + * ``measure_coverage`` -> run ``pytest --cov=src`` in the per-branch + worktree (offline) -> line coverage ``%`` or + ``None`` on tool error. + * ``compute_coverage_verdict`` -> pure: compare (measured, baseline, floor) under + a policy + epsilon -> ``(ok, reason)``. + * ``write_coverage_report`` / ``parse_coverage_status`` -> write the + ``18-coverage-report.md`` artefact and read its machine verdict back (single + source of truth: the gate returns exactly the frontmatter it wrote, AC-9). + * ``ratchet_baseline_on_merge`` -> on a CONFIRMED merge (``_handle_merge_verify``, + ``deploy -> done`` edge) raise the per-repo baseline UP from the merged branch's + measured coverage (atomic compare-and-set, never decreases — FR-4 / D5). + * ``check_coverage_gate`` -> the orchestrating entry the QG wrapper delegates + to. + +Invariants (ADR-001 §7, never broken): + * **Tool error -> fail-open + WARNING by default** (FR-6/AC-6): a coverage-tool + failure / unparseable metric degrades fail-open (anti-loop, precedent + ORCH-061/022 dep-audit); ``coverage_tool_fail_closed`` flips it to strict. + * **never-raise** (AC-7): any internal error is swallowed; an exception never + escapes into ``advance_stage``. + * **Baseline never decreases** (FR-4): the ratchet is an atomic SQL compare-and-set + under the held merge-lease (ORCH-043), so two parallel merges can never lower or + lose the value. + * **Self-hosting safety** (AC-7): the gate only measures / reads / writes the + artefact / decides. It never calls the deploy hook, never restarts the prod + container, never pushes / force-pushes ``main``. + +This module is a **leaf**: it imports only ``config`` / ``git_worktree`` and lazily +``qg.checks.is_self_hosting_repo`` / ``db`` / ``notifications``; it never imports +``stage_engine``. +""" + +import json +import logging +import os +import subprocess + +from .config import settings +from .git_worktree import ensure_worktree, get_worktree_path + +logger = logging.getLogger("orchestrator.coverage_gate") + + +# --------------------------------------------------------------------------- +# Conditionality (mirrors security_gate_applies / _merge_gate_applies) +# --------------------------------------------------------------------------- +def coverage_gate_applies(repo: str) -> bool: + """Whether the coverage-gate is REAL for this repo (conditional rollout). + + Mirrors the ORCH-22 / ORCH-43 / ORCH-58 pattern: + * ``coverage_gate_enabled=False`` -> always False (kill-switch; pipeline is + 1:1 as before ORCH-027 for everyone). + * ``coverage_gate_repos`` (CSV) non-empty -> real only for the listed repos. + * empty CSV -> real ONLY for the self-hosting repo (``orchestrator``). + Never raises (AC-7): any error -> False (the safe no-op default). + """ + try: + if not settings.coverage_gate_enabled: + return False + raw = (settings.coverage_gate_repos or "").strip() + if raw: + allowed = {r.strip().lower() for r in raw.split(",") if r.strip()} + return (repo or "").strip().lower() in allowed + # Lazy import keeps this module a leaf (no qg import at module load). + from .qg.checks import is_self_hosting_repo + return is_self_hosting_repo(repo) + except Exception as e: # noqa: BLE001 - never-raise contract + logger.warning("coverage_gate_applies error for %s: %s", repo, e) + return False + + +# --------------------------------------------------------------------------- +# Measurement (pytest --cov=src in the per-branch worktree) — FR-1 / D2 +# --------------------------------------------------------------------------- +def parse_coverage_percent(data) -> float | None: + """Pure: extract ``totals.percent_covered`` (line coverage ``%``) from a + coverage.py JSON dict. Returns ``None`` if the shape is missing / unparseable. + Never raises. + """ + try: + if not isinstance(data, dict): + return None + totals = data.get("totals") + if not isinstance(totals, dict): + return None + pct = totals.get("percent_covered") + if pct is None: + return None + return float(pct) + except (TypeError, ValueError): + return None + + +def measure_coverage(repo: str, branch: str) -> float | None: + """Run ``pytest --cov=src`` in the per-branch worktree -> line coverage ``%``. + + Scope is ``src/`` only (the tests themselves are out of scope, BRD §«Вне + объёма»). Offline — coverage needs no network. The measurer is intentionally + encapsulated here so the pure decision logic and the baseline storage are + stack-agnostic (a future jest/jacoco measurer is a new ``measure_*`` branch, + BR-6). + + The coverage metric is read from the ``--cov-report=json`` file regardless of + the pytest exit code: a non-zero exit because of *failing tests* is already + caught upstream (``check_ci_green`` / merge-gate re-test), and a partial run + still produces a meaningful coverage JSON. A genuine tool error (missing + plugin / timeout / no JSON / unparseable) -> ``None`` (the caller degrades + fail-open by default, FR-6). Never raises (AC-7). + """ + try: + wt = ensure_worktree(repo, branch) + except Exception as e: # noqa: BLE001 - never-raise contract + logger.warning("measure_coverage: worktree error for %s/%s: %s", repo, branch, e) + return None + + cov_json = os.path.join(wt, ".coverage-report.json") + # Remove a stale report so we never read a previous pass's metric. + try: + if os.path.isfile(cov_json): + os.remove(cov_json) + except OSError: + pass + + cmd = [ + "python", "-m", "pytest", "tests/", + "--cov=src", + f"--cov-report=json:{cov_json}", + "--cov-report=", # suppress the terminal cov report (json only) + "-q", + ] + timeout = settings.coverage_run_timeout_s + try: + subprocess.run(cmd, cwd=wt, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + logger.warning( + "measure_coverage: pytest --cov timed out after %ss for %s/%s", + timeout, repo, branch, + ) + return None + except FileNotFoundError: + logger.warning( + "measure_coverage: pytest / pytest-cov not available for %s/%s", repo, branch + ) + return None + except (subprocess.SubprocessError, OSError) as e: + logger.warning("measure_coverage: pytest --cov error for %s/%s: %s", repo, branch, e) + return None + + data = None + try: + if not os.path.isfile(cov_json): + logger.warning( + "measure_coverage: no coverage json produced for %s/%s", repo, branch + ) + return None + with open(cov_json, "r", encoding="utf-8") as f: + data = json.load(f) + except (OSError, ValueError) as e: + logger.warning( + "measure_coverage: cannot parse coverage json for %s/%s: %s", repo, branch, e + ) + return None + finally: + try: + if os.path.isfile(cov_json): + os.remove(cov_json) + except OSError: + pass + + return parse_coverage_percent(data) + + +# --------------------------------------------------------------------------- +# Pure decision (FR-2 / D3) — the core of the unit tests +# --------------------------------------------------------------------------- +def compute_coverage_verdict(measured, baseline, floor, policy, epsilon) -> tuple[bool, str]: + """Pure: decide PASS/FAIL from (measured, baseline, floor, policy, epsilon). + + Deterministic, no LLM, no I/O. Returns ``(ok: bool, reason: str)``. + + * ``policy = "absolute"`` -> PASS ⇔ ``measured >= floor - epsilon``. + * ``policy = "baseline"`` -> PASS ⇔ ``measured >= baseline - epsilon``. + * ``policy = "both"`` (default) -> PASS ⇔ BOTH conditions hold. + * ``baseline is None`` (no stored baseline / bootstrap) -> the baseline + condition does NOT apply (cannot regress against nothing); only the + absolute part decides. For ``policy = "baseline"`` with no baseline this is + a bootstrap PASS (the measured value seeds the baseline at merge, D5). + * ``epsilon`` — a small non-negative tolerance so jitter at the boundary does + not bounce a task (NFR-4). + + Never raises: bad inputs -> ``(False, reason)`` (a verdict cannot be computed -> + conservative FAIL for the pure function; the orchestrating entry maps a *tool* + error to fail-open separately). + """ + try: + pol = (policy or "both").strip().lower() + eps = max(0.0, float(epsilon if epsilon is not None else 0.0)) + m = float(measured) + except (TypeError, ValueError) as e: + return False, f"coverage verdict: bad inputs ({e})" + + abs_applicable = pol in ("absolute", "both") + base_applicable = pol in ("baseline", "both") and baseline is not None + + checks: list[str] = [] + ok = True + + if abs_applicable: + try: + f = float(floor if floor is not None else 0.0) + except (TypeError, ValueError): + f = 0.0 + abs_ok = m >= f - eps + checks.append( + f"absolute {m:.2f}% >= floor {f:.2f}%-eps{eps:.2f} -> " + f"{'PASS' if abs_ok else 'FAIL'}" + ) + ok = ok and abs_ok + + if base_applicable: + b = float(baseline) + base_ok = m >= b - eps + checks.append( + f"baseline {m:.2f}% >= base {b:.2f}%-eps{eps:.2f} -> " + f"{'PASS' if base_ok else 'FAIL'}" + ) + ok = ok and base_ok + elif pol in ("baseline", "both") and baseline is None: + checks.append("baseline N/A (bootstrap — no stored baseline)") + + body = "; ".join(checks) if checks else "no applicable condition (bootstrap) -> PASS" + reason = f"measured={m:.2f}% policy={pol} eps={eps:.2f}: {body}" + return ok, reason + + +def compute_delta(measured, baseline, floor) -> float: + """Pure: signed ``measured - max(applicable references)`` (%, 2 decimals). + + References are the present ones among ``baseline`` / ``floor``. With neither -> + ``0.0``. Never raises. + """ + try: + m = float(measured) + refs = [] + if baseline is not None: + refs.append(float(baseline)) + if floor is not None: + refs.append(float(floor)) + if not refs: + return 0.0 + return round(m - max(refs), 2) + except (TypeError, ValueError): + return 0.0 + + +# --------------------------------------------------------------------------- +# Artefact: write the report, read the machine verdict back (FR-7 / D7 / AC-9) +# --------------------------------------------------------------------------- +def _report_rel(work_item_id: str) -> str: + return f"docs/work-items/{work_item_id}/18-coverage-report.md" + + +def _report_path(repo: str, work_item_id: str, branch: str) -> str: + """Absolute path of 18-coverage-report.md inside the task worktree.""" + try: + wt = get_worktree_path(repo, branch) + if not os.path.isdir(wt): + wt = ensure_worktree(repo, branch) + except Exception: # noqa: BLE001 - never-raise; fall back to shared clone + wt = os.path.join(settings.repos_dir, repo) + return os.path.join(wt, _report_rel(work_item_id)) + + +def _num(v) -> str: + """Render a numeric field with 2 decimals, or empty for None/unparseable.""" + if v is None: + return "" + try: + return f"{float(v):.2f}" + except (TypeError, ValueError): + return "" + + +def render_coverage_report(work_item_id: str, fields: dict) -> str: + """Pure: render the 18-coverage-report.md content (frontmatter + body). + + The machine verdict lives ONLY in the YAML frontmatter ``coverage_status:`` + (canon, regiser-sensitive); ``measured_coverage`` is the single source of truth + for the ratchet (D5). Never raises. + """ + baseline = fields.get("baseline") + baseline_str = "" if baseline is None else _num(baseline) + return ( + "---\n" + f"coverage_status: {fields.get('coverage_status', 'FAIL')}\n" + f"work_item: {work_item_id}\n" + f"measured_coverage: {_num(fields.get('measured_coverage'))}\n" + f"baseline: {baseline_str}\n" + f"floor: {_num(fields.get('floor'))}\n" + f"policy: {fields.get('policy', 'both')}\n" + f"epsilon: {_num(fields.get('epsilon'))}\n" + f"delta: {_num(fields.get('delta'))}\n" + "---\n" + f"# Coverage Report — {work_item_id}\n\n" + "Детерминированный гейт покрытия (ORCH-027) — под-гейт ребра " + "`deploy-staging→deploy` (ПОСЛЕ merge-gate, ДО image-freshness). Машинный " + "вердикт читается ТОЛЬКО из `coverage_status:` frontmatter выше.\n\n" + "## Verdict\n" + f"{fields.get('reason', '')}\n\n" + "## Measurement\n" + f"{fields.get('measurement', '')}\n\n" + "## Policy\n" + f"{fields.get('policy_detail', '')}\n" + ) + + +def write_coverage_report(repo: str, work_item_id: str, branch: str, fields: dict) -> str: + """Write 18-coverage-report.md into the task worktree; return its path. + + Best-effort / never-raise: a write error is logged and the path is still + returned (the caller's read-back then fails closed).""" + path = _report_path(repo, work_item_id, branch) + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(render_coverage_report(work_item_id, fields)) + except OSError as e: + logger.error("write_coverage_report error for %s/%s: %s", repo, work_item_id, e) + return path + + +def parse_coverage_status(content: str) -> tuple[bool, str]: + """Map a 18-coverage-report.md body to a quality-gate verdict by reading ONLY + the machine-readable ``coverage_status:`` YAML frontmatter — never the prose. + + Mirrors ``parse_security_status`` (canon: machine verdict only from frontmatter, + AC-9). The negative token (FAIL) is authoritative (checked first). Returns: + * ``coverage_status: PASS`` -> ``(True, "Coverage status: PASS")`` + * ``coverage_status: FAIL`` -> ``(False, "Coverage status: FAIL")`` + * missing field / no frontmatter / bad YAML -> ``(False, )``. + + Parse delegated to the unified ``frontmatter.parse_frontmatter`` primitive + (ORCH-052c single source of YAML-frontmatter logic). + """ + from .frontmatter import parse_frontmatter + + parse = parse_frontmatter(content) + if parse.yaml_error is not None: + return False, f"Invalid YAML frontmatter in coverage report: {parse.yaml_error}" + status = None + if parse.has_block and not parse.malformed: + status = str(parse.data.get("coverage_status", "")).upper().strip() + if status == "FAIL": + return False, "Coverage status: FAIL" + if status == "PASS": + return True, "Coverage status: PASS" + return False, f"No machine-readable coverage_status in frontmatter (got: {status!r})" + + +def read_measured_coverage(content: str) -> float | None: + """Read ``measured_coverage`` (%, float) from a 18-coverage-report.md body via + the unified frontmatter parser. ``None`` when absent / unparseable (ratchet then + no-ops). Never raises. + """ + try: + from .frontmatter import parse_frontmatter + parse = parse_frontmatter(content) + if not parse.has_block or parse.malformed: + return None + raw = parse.data.get("measured_coverage") + if raw is None or (isinstance(raw, str) and not raw.strip()): + return None + return float(raw) + except (TypeError, ValueError): + return None + except Exception as e: # noqa: BLE001 - never-raise + logger.warning("read_measured_coverage error: %s", e) + return None + + +def _error_fields(work_item_id, floor, policy, epsilon, baseline, *, fail_closed: bool) -> dict: + """Build the report fields for a tool-error pass (FR-6).""" + status = "FAIL" if fail_closed else "PASS" + mode = "fail-closed (FAIL)" if fail_closed else "fail-open (WARNING)" + return { + "coverage_status": status, + "measured_coverage": None, + "baseline": baseline, + "floor": floor, + "policy": policy, + "epsilon": epsilon, + "delta": None, + "reason": f"coverage measurement failed -> {mode}", + "measurement": ( + "coverage tool error / unparseable metric " + f"(coverage_tool_fail_closed={fail_closed})" + ), + "policy_detail": f"policy={policy}, floor={floor}, baseline={baseline}, epsilon={epsilon}", + } + + +# --------------------------------------------------------------------------- +# Ratchet baseline UP on a confirmed merge (FR-4 / D5) +# --------------------------------------------------------------------------- +def ratchet_baseline_on_merge(repo: str, work_item_id: str, branch: str, sha: str | None = None) -> bool: + """Raise the per-repo coverage baseline UP from the merged branch's measured + coverage. Called from ``_handle_merge_verify`` (deploy -> done edge) AFTER the + merge is confirmed and BEFORE the task advances to ``done`` (D5). + + Reads the measured value from ``18-coverage-report.md`` (single source of truth + — the exact metric the gate wrote on the deploy-staging->deploy edge) and applies + an atomic compare-and-set (``db.ratchet_coverage_baseline``) that never lowers + the baseline. Bootstrap: the first applicable merge seeds the baseline. + + Returns True iff the baseline was inserted/raised. never-raise (AC-7): any error + -> False (observability best-effort; a ratchet failure must never break the + deploy->done path). + """ + try: + if not coverage_gate_applies(repo): + return False + path = _report_path(repo, work_item_id, branch) + try: + with open(path, "r", encoding="utf-8") as f: + content = f.read() + except OSError as e: + logger.warning( + "ratchet: cannot read coverage report for %s/%s: %s", repo, work_item_id, e + ) + return False + measured = read_measured_coverage(content) + if measured is None: + logger.warning( + "ratchet: no measured_coverage in report for %s/%s", repo, work_item_id + ) + return False + from . import db + updated = db.ratchet_coverage_baseline(repo, measured, sha) + if updated: + logger.info( + "coverage baseline ratcheted for %s -> %.2f%% (sha=%s)", repo, measured, sha + ) + else: + logger.info( + "coverage baseline unchanged for %s (measured %.2f%% not above current)", + repo, measured, + ) + return updated + except Exception as e: # noqa: BLE001 - never-raise contract + logger.error("ratchet_baseline_on_merge error for %s/%s: %s", repo, work_item_id, e) + return False + + +# --------------------------------------------------------------------------- +# Orchestrating entry — delegated to by qg.checks.check_coverage_gate +# --------------------------------------------------------------------------- +def check_coverage_gate(repo: str, work_item_id: str, branch: str) -> tuple[bool, str]: + """ORCH-027 coverage-gate on the deploy-staging -> deploy edge (after merge-gate). + + Deterministic, no LLM. Algorithm (ADR-001 D1..D7): + 1. Conditionality: ``coverage_gate_enabled=False`` -> ``(True, "...disabled")``; + a repo the gate is not real for -> ``(True, "coverage-gate N/A for ")``. + 2. ``measure_coverage`` (pytest --cov=src in the worktree). ``None`` (tool + error) -> fail-open + WARNING by default (``coverage_tool_fail_closed`` + flips to FAIL), FR-6. + 3. ``compute_coverage_verdict`` -> write ``18-coverage-report.md`` -> read the + verdict BACK via ``parse_coverage_status`` (single source of truth: the + returned verdict == the artefact frontmatter, AC-9). + 4. FAIL -> ``(False, reason)`` (engine rolls back to ``development`` + releases + the merge lease); PASS -> ``(True, reason)`` (engine proceeds to + image-freshness). + + Never-raise (AC-7): any internal error -> a (bool, reason) pair following the + fail-open default (so an unexpected fault never wedges the autonomous pipeline), + unless ``coverage_tool_fail_closed`` is set. + """ + floor = getattr(settings, "coverage_min_percent", 0.0) + policy = getattr(settings, "coverage_policy", "both") + epsilon = getattr(settings, "coverage_epsilon", 0.5) + try: + if not settings.coverage_gate_enabled: + return True, "coverage-gate disabled" + if not coverage_gate_applies(repo): + return True, f"coverage-gate N/A for {repo}" + + from . import db + try: + baseline = db.get_coverage_baseline(repo) + except Exception as e: # noqa: BLE001 - baseline read best-effort + logger.warning("coverage-gate: baseline read error for %s: %s", repo, e) + baseline = None + + measured = measure_coverage(repo, branch) + if measured is None: + fail_closed = bool(settings.coverage_tool_fail_closed) + fields = _error_fields( + work_item_id, floor, policy, epsilon, baseline, fail_closed=fail_closed + ) + write_coverage_report(repo, work_item_id, branch, fields) + if fail_closed: + logger.warning( + "coverage-gate %s/%s: measurement failed -> fail-CLOSED (FAIL)", + repo, work_item_id, + ) + return False, "coverage-gate fail-closed: measurement failed (tool error)" + logger.warning( + "coverage-gate %s/%s: measurement failed -> fail-OPEN + WARNING", + repo, work_item_id, + ) + return True, "coverage-gate fail-open (WARNING): measurement failed (tool error)" + + ok, reason = compute_coverage_verdict(measured, baseline, floor, policy, epsilon) + delta = compute_delta(measured, baseline, floor) + fields = { + "coverage_status": "PASS" if ok else "FAIL", + "measured_coverage": measured, + "baseline": baseline, + "floor": floor, + "policy": policy, + "epsilon": epsilon, + "delta": delta, + "reason": reason, + "measurement": f"pytest --cov=src: line coverage src/ = {measured:.2f}%", + "policy_detail": ( + f"policy={policy}, floor={floor}%, " + f"baseline={'bootstrap' if baseline is None else f'{baseline:.2f}%'}, " + f"epsilon={epsilon}%" + ), + } + path = write_coverage_report(repo, work_item_id, branch, fields) + + # Read the machine verdict back from the artefact we just wrote — so the + # returned (bool, reason) is guaranteed == the YAML frontmatter (AC-9). + try: + with open(path, "r", encoding="utf-8") as f: + content = f.read() + except OSError as e: + return False, f"cannot read coverage report (fail-closed): {e}" + verdict_ok, _v = parse_coverage_status(content) + + if verdict_ok: + logger.info("coverage-gate passed for %s/%s: %s", repo, work_item_id, reason) + return True, f"coverage OK ({reason})" + + # FAIL -> surface loudly (Telegram with the clickable issue number, FR-7). + try: + from .notifications import send_telegram, link_for + base_str = "n/a" if baseline is None else f"{baseline:.2f}%" + send_telegram( + f"\U0001f4c9 {link_for(work_item_id)}: coverage-гейт FAIL — измерено " + f"{measured:.2f}% (floor {floor}%, baseline {base_str}, " + f"delta {delta:+.2f}%). Откат на development для доработки тестов." + ) + except Exception as e: # noqa: BLE001 - telegram best-effort + logger.warning("coverage-gate FAIL telegram failed: %s", e) + return False, reason + except Exception as e: # noqa: BLE001 - never-raise contract (AC-7) + logger.error("check_coverage_gate error for %s/%s: %s", repo, branch, e) + # An unexpected internal error follows the fail-open default (anti-loop): a + # coverage-tool/logic fault must not wedge the autonomous pipeline. The + # operator can flip coverage_tool_fail_closed to make it strict. + try: + if settings.coverage_tool_fail_closed: + return False, f"coverage-gate error (fail-closed): {e}" + except Exception: # noqa: BLE001 + pass + return True, f"coverage-gate error (fail-open): {e}" + + +# --------------------------------------------------------------------------- +# Observability snapshot for GET /queue (FR-7 / AC-9) +# --------------------------------------------------------------------------- +def snapshot() -> dict: + """Read-only coverage-gate summary for GET /queue (FR-7 / AC-9). + + Additive block; existing /queue keys are untouched. never-raise: any error -> + a minimal dict with the flags. + """ + try: + enabled = bool(settings.coverage_gate_enabled) + except Exception: # noqa: BLE001 + enabled = False + out = { + "enabled": enabled, + "repos": getattr(settings, "coverage_gate_repos", "") or "", + "policy": getattr(settings, "coverage_policy", "both"), + "floor": getattr(settings, "coverage_min_percent", 0.0), + "epsilon": getattr(settings, "coverage_epsilon", 0.5), + "fail_closed": bool(getattr(settings, "coverage_tool_fail_closed", False)), + "baselines": {}, + } + try: + from . import db + out["baselines"] = db.all_coverage_baselines() + except Exception as e: # noqa: BLE001 - never-raise -> empty baselines + logger.warning("coverage snapshot baselines error: %s", e) + return out diff --git a/src/db.py b/src/db.py index 6701034..3c4c56f 100644 --- a/src/db.py +++ b/src/db.py @@ -199,10 +199,138 @@ def init_db(): CREATE INDEX IF NOT EXISTS idx_repo_freeze_active ON repo_freeze (repo, cleared_at); """) + # ORCH-027 (FR-4, ADR-001 D4): additive per-repo coverage baseline for the + # coverage-gate ratchet. One row per repo; the baseline is monotonically + # non-decreasing via ratchet_coverage_baseline (atomic compare-and-set). Purely + # ADDITIVE (CREATE TABLE IF NOT EXISTS, pattern repo_freeze/job_deps) -> + # idempotent, restart-safe on the shared prod DB; existing tables untouched + # (NFR-5). See docs/work-items/ORCH-027/08-data-requirements.md. + conn.executescript(""" + CREATE TABLE IF NOT EXISTS coverage_baseline ( + repo TEXT PRIMARY KEY, + coverage REAL NOT NULL, + source_sha TEXT, + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + """) conn.commit() conn.close() +def get_coverage_baseline(repo: str) -> float | None: + """ORCH-027: read the per-repo coverage baseline (%, line coverage). + + Returns ``None`` when no baseline is stored yet (bootstrap mode — the gate then + decides on the absolute floor only, D3). Raises only on a real DB error (the + coverage_gate leaf caller wraps this in its never-raise contract). + """ + if not repo: + return None + conn = get_db() + try: + row = conn.execute( + "SELECT coverage FROM coverage_baseline WHERE repo = ?", (repo,) + ).fetchone() + finally: + conn.close() + if row is None: + return None + try: + return float(row["coverage"]) + except (TypeError, ValueError): + return None + + +def ratchet_coverage_baseline(repo: str, coverage: float, sha: str | None = None) -> bool: + """ORCH-027 (FR-4, D5): raise the per-repo coverage baseline UP, never down. + + Atomic compare-and-set: ``UPDATE ... WHERE coverage <= ?`` (the baseline never + decreases — an equal value is an idempotent no-harm re-stamp), or ``INSERT`` when + no row exists yet (bootstrap). Under the held merge-lease (ORCH-043) plus this + single-statement guard, two parallel merges can never lower or lose the value. + Returns True iff a row was inserted or raised. + """ + if not repo: + return False + try: + cov = float(coverage) + except (TypeError, ValueError): + return False + conn = get_db() + try: + cur = conn.execute( + "UPDATE coverage_baseline " + "SET coverage = ?, source_sha = ?, updated_at = datetime('now') " + "WHERE repo = ? AND coverage <= ?", + (cov, sha, repo, cov), + ) + changed = cur.rowcount or 0 + if changed == 0: + # No row updated: either the row is absent (bootstrap INSERT) or the + # existing baseline is already higher (skip — never lower it). + exists = conn.execute( + "SELECT 1 FROM coverage_baseline WHERE repo = ?", (repo,) + ).fetchone() + if exists is None: + conn.execute( + "INSERT INTO coverage_baseline (repo, coverage, source_sha, updated_at) " + "VALUES (?, ?, ?, datetime('now'))", + (repo, cov, sha), + ) + changed = 1 + conn.commit() + return bool(changed) + finally: + conn.close() + + +def set_coverage_baseline(repo: str, coverage: float, sha: str | None = None) -> bool: + """ORCH-027 (D8): UNCONDITIONALLY set the per-repo coverage baseline. + + For a legitimate one-off coverage drop (e.g. removing a large tested module) via + the manual ``POST /coverage/baseline`` override. Unlike ``ratchet_coverage_baseline`` + this CAN lower the baseline. Returns True on success. + """ + if not repo: + return False + try: + cov = float(coverage) + except (TypeError, ValueError): + return False + conn = get_db() + try: + conn.execute( + "INSERT INTO coverage_baseline (repo, coverage, source_sha, updated_at) " + "VALUES (?, ?, ?, datetime('now')) " + "ON CONFLICT(repo) DO UPDATE SET coverage = excluded.coverage, " + "source_sha = excluded.source_sha, updated_at = excluded.updated_at", + (repo, cov, sha), + ) + conn.commit() + return True + finally: + conn.close() + + +def all_coverage_baselines() -> dict: + """ORCH-027: all per-repo coverage baselines for the GET /queue snapshot.""" + conn = get_db() + try: + rows = conn.execute( + "SELECT repo, coverage, source_sha, updated_at FROM coverage_baseline" + ).fetchall() + finally: + conn.close() + return { + r["repo"]: { + "coverage": r["coverage"], + "source_sha": r["source_sha"], + "updated_at": r["updated_at"], + } + for r in rows + } + + def _ensure_column(conn, table: str, column: str, decl: str): """Add a column to `table` if it does not already exist (idempotent migration).""" cols = [r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()] diff --git a/src/main.py b/src/main.py index fdd9fa2..38840bc 100644 --- a/src/main.py +++ b/src/main.py @@ -170,6 +170,7 @@ async def queue(): from . import merge_gate from . import task_deps from . import serial_gate + from . import coverage_gate from . import labels from . import cancel from .disk_watchdog import disk_watchdog @@ -189,6 +190,9 @@ async def queue(): # ORCH-088 (D9 / AC-10): per-repo serial-gate observability (read-only) — # active task, queued/waiting analyst-jobs, freeze state. Additive block. "serial_gate": serial_gate.snapshot(), + # ORCH-027 (FR-7 / AC-9): coverage-gate observability (read-only) — + # kill-switch, scope, policy/floor/epsilon, per-repo baselines. Additive block. + "coverage": coverage_gate.snapshot(), # ORCH-089 (D7): auto-mode-by-label observability (read-only) — kill-switch, # label names, scope. Additive block. "auto_labels": labels.snapshot(), @@ -236,3 +240,23 @@ async def serial_gate_unfreeze(repo: str = ""): except Exception: pass return {"ok": True, "repo": repo, "cleared": cleared, "frozen": frozen} + + +@app.post("/coverage/baseline") +async def coverage_set_baseline(repo: str = "", value: float | None = None): + """ORCH-027 (D8): manually set/override the per-repo coverage baseline. + + For a legitimate one-off coverage drop (e.g. removing a large tested module) the + operator sets the baseline directly here (by образцу ``POST /serial-gate/unfreeze``) + instead of waiting for the upward-only ratchet. Unlike the ratchet this CAN lower + the baseline. Alternative without this endpoint: temporarily flip + ``ORCH_COVERAGE_POLICY=absolute``. + """ + from . import db + if not repo or not repo.strip(): + return {"ok": False, "error": "missing 'repo'", "repo": repo} + if value is None: + return {"ok": False, "error": "missing 'value'", "repo": repo} + repo = repo.strip() + ok = db.set_coverage_baseline(repo, value, sha="manual-override") + return {"ok": ok, "repo": repo, "baseline": db.get_coverage_baseline(repo)} diff --git a/src/qg/checks.py b/src/qg/checks.py index d4f77c2..ca63fd6 100644 --- a/src/qg/checks.py +++ b/src/qg/checks.py @@ -755,6 +755,23 @@ def check_security_gate(repo: str, work_item_id: str, branch: str) -> tuple[bool return _impl(repo, work_item_id, branch) +def check_coverage_gate(repo: str, work_item_id: str, branch: str) -> tuple[bool, str]: + """ORCH-027 coverage sub-gate (pytest --cov=src) on the deploy-staging -> deploy + edge, run AFTER the merge-gate (caught-up HEAD) and BEFORE image-freshness. + + Thin registry wrapper that delegates to ``coverage_gate.check_coverage_gate`` + (measure line coverage of src/, compare to floor/baseline under a policy, write/ + read-back ``18-coverage-report.md``). The real logic lives in + ``src/coverage_gate.py`` (leaf module, never-raise, fail-open on a tool error by + default); importing it lazily here avoids an import cycle (coverage_gate imports + is_self_hosting_repo from this module). For non-self repos with an empty scope it + returns ``(True, "coverage-gate N/A for ")`` so the deploy edge is unchanged + for them (AC-5). + """ + from ..coverage_gate import check_coverage_gate as _impl + return _impl(repo, work_item_id, branch) + + # Registry for dynamic lookup by name QG_CHECKS = { "check_analysis_approved": check_analysis_approved, @@ -770,4 +787,5 @@ QG_CHECKS = { "check_branch_mergeable": check_branch_mergeable, "check_staging_image_fresh": _check_staging_image_fresh, "check_security_gate": check_security_gate, + "check_coverage_gate": check_coverage_gate, } diff --git a/src/stage_engine.py b/src/stage_engine.py index 54f1582..1eb9a33 100644 --- a/src/stage_engine.py +++ b/src/stage_engine.py @@ -322,6 +322,19 @@ def advance_stage( ): return result + # --- ORCH-027 coverage sub-gate (deploy-staging -> deploy edge) ---- + # AFTER the merge-gate (coverage measured on the caught-up HEAD that + # lands in `main`, so the metric matches landed code) and BEFORE the + # image-freshness rebuild (fail before the expensive docker rebuild). + # Deterministic (no LLM): pytest --cov=src -> line coverage % vs floor / + # ratchet baseline. FAIL -> rollback to development + release the merge + # lease (held by the merge-gate's PASS). It owns the outcome on + # intervention (mirrors the merge-gate / image-freshness). + if _handle_coverage_gate( + task_id, current_stage, repo, work_item_id, branch, agent, result + ): + return result + # --- ORCH-058 freshness sub-gate (deploy-staging -> deploy edge) --- # AFTER the merge-gate finalised the validated HEAD and BEFORE Phase A. # Rebuilds the staging image from that validated commit + recreates 8501 @@ -1124,6 +1137,90 @@ def _handle_security_gate( return True +# --------------------------------------------------------------------------- +# ORCH-027: coverage sub-gate on the deploy-staging -> deploy edge +# --------------------------------------------------------------------------- +def _handle_coverage_gate( + task_id, current_stage, repo, work_item_id, branch, agent, result: AdvanceResult +) -> bool: + """Run check_coverage_gate on the deploy-staging -> deploy edge (ORCH-027). + + Runs AFTER the merge-gate (so coverage is measured on the rebased/caught-up HEAD + that actually lands in `main`) and BEFORE the image-freshness rebuild (fail before + the expensive docker rebuild). Deterministic (no LLM): pytest --cov=src in the + per-branch worktree -> line coverage % -> compute_coverage_verdict vs the absolute + floor and/or the ratchet baseline. The machine verdict lives in + 18-coverage-report.md frontmatter. A coverage-tool error degrades fail-open + + WARNING by default (FR-6), so an infra hiccup never wedges the autonomous pipeline. + + Returns True if the gate INTERVENED (the caller must return without advancing): + * FAIL (coverage below policy) -> ROLLBACK to development (+ developer retry, + capped by MAX_DEVELOPER_RETRIES) and RELEASE the merge lease (the merge-gate + held it on its PASS; coverage failed before the merge — mirrors the + image-freshness rollback, ADR-001 D1/TR-2). + Returns False when the gate PASSED (clean / fail-open / N/A) so advance_stage + proceeds to the image-freshness sub-gate. On a PASS the merge lease stays HELD + until the actual merge (released on done / rollback). + """ + passed, reason = _run_qg("check_coverage_gate", repo, work_item_id, branch) + if passed: + logger.info(f"Task {task_id}: coverage-gate passed ({reason})") + return False + + result.qg_name = "check_coverage_gate" + result.qg_passed = False + result.qg_reason = reason + + update_task_stage(task_id, "development") + notify_stage_change(task_id, current_stage, "development") + plane_notify_stage(work_item_id, current_stage, "development") + result.rolled_back_to = "development" + set_issue_in_progress(work_item_id) + # The merge-gate held the lease on its PASS; coverage failed before the merge, so + # release it (holder-aware no-op if a different task already owns it). Mirrors the + # image-freshness rollback (ADR-001 D1/TR-2). + try: + merge_gate.release_merge_lease(repo, branch) + except Exception as e: # noqa: BLE001 - defensive + logger.warning(f"Task {task_id}: merge-lease release on coverage fail failed: {e}") + notify_qg_failure(task_id, current_stage, "check_coverage_gate", reason) + plane_add_comment( + work_item_id, + f"❌ Coverage-гейт провален ({reason}). Откат на development. " + f"Developer нужен для добавления тестов (покрытие src/ просело).", + author="deployer", + ) + retry_count = _developer_retry_count(task_id) + if retry_count < MAX_DEVELOPER_RETRIES: + report_ref = f"docs/work-items/{work_item_id}/18-coverage-report.md" + task_desc = ( + f"Work item: {work_item_id}\nRepo: {repo}\nBranch: {branch}\n" + f"Stage: development\nNote: Coverage-гейт провален " + f"(attempt {retry_count + 1}/{MAX_DEVELOPER_RETRIES}). " + f"Причина: {reason}. Добавь тесты, чтобы покрытие src/ не падало ниже " + f"политики. Полный отчёт: {report_ref}" + ) + new_job = enqueue_job("developer", repo, task_desc, task_id=task_id) + result.enqueued_agent = "developer" + result.enqueued_job_id = new_job + logger.info( + f"Task {task_id}: coverage-gate FAILED, enqueued developer (job_id={new_job})" + ) + else: + set_issue_blocked(work_item_id) + send_telegram( + f"\U0001f6a8 {link_for(work_item_id)}: Coverage-гейт still failing after " + f"{MAX_DEVELOPER_RETRIES} developer retries ({reason}). " + f"Manual intervention needed." + ) + result.alerted = True + logger.error( + f"Task {task_id}: coverage-gate FAILED, rolled back deploy-staging -> " + f"development ({reason})" + ) + return True + + # --------------------------------------------------------------------------- # ORCH-058: staging-image freshness sub-gate on the deploy-staging -> deploy edge # --------------------------------------------------------------------------- @@ -1546,6 +1643,17 @@ def _handle_merge_verify(task_id, repo, work_item_id, branch, result: AdvanceRes task_id, repo, work_item_id, branch, guard_msg, result ) + # ORCH-027 (D5): ratchet the per-repo coverage baseline UP from this + # merged branch's measured coverage (single source of truth: + # 18-coverage-report.md). Atomic compare-and-set under the still-held + # merge-lease -> the baseline never decreases. never-raise (observability + # best-effort): a ratchet failure must never break the deploy->done path. + try: + from . import coverage_gate + coverage_gate.ratchet_baseline_on_merge(repo, work_item_id, branch, sha) + except Exception as e: # noqa: BLE001 - observability best-effort + logger.warning(f"Task {task_id}: coverage baseline ratchet failed: {e}") + merge_gate.note_merge_verified() try: self_deploy.record_merged_to_main(repo, work_item_id, branch, True) diff --git a/tests/test_config.py b/tests/test_config.py index 581f0f3..b43a179 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -248,6 +248,7 @@ def test_tc19_qg_checks_registry_unchanged(): "check_branch_mergeable", "check_staging_image_fresh", "check_security_gate", + "check_coverage_gate", } diff --git a/tests/test_coverage_gate.py b/tests/test_coverage_gate.py new file mode 100644 index 0000000..d73dfb4 --- /dev/null +++ b/tests/test_coverage_gate.py @@ -0,0 +1,471 @@ +"""ORCH-027 / TC-01..TC-15: the coverage-gate leaf module (src/coverage_gate.py). + +These exercise the DETERMINISTIC core: the pure verdict / delta / frontmatter +helpers (no binaries needed), the ratchet baseline against a real tmp SQLite DB, +the conditionality / kill-switch / fail-open behaviour with the measurer mocked, +never-raise, and the gate's integration into advance_stage / GET /queue. + +Contract under test (ADR-001 §7): + * the verdict is a deterministic pure function of (measured, baseline, floor, + policy, epsilon) — no LLM, all border / epsilon cases covered; + * the ratchet baseline only moves UP and bootstraps on the first merge; + * conditionality: empty scope -> self-hosting only; out-of-scope -> no-op N/A; + kill-switch off -> inert; + * a coverage-tool error degrades fail-open + WARNING by default, fail-closed only + when configured; + * the machine verdict lives ONLY in the YAML frontmatter (read-back == written); + * never-raise: any internal error -> a (bool, reason) pair, no exception escapes; + * self-hosting safety: the gate never deploys / restarts prod / pushes main. +""" +import os +import tempfile + +os.environ["ORCH_DB_PATH"] = os.path.join(tempfile.gettempdir(), "test_coverage_gate.db") +os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") +os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") + +import pytest # noqa: E402 + +import src.db as db # noqa: E402 +from src import config as cfg # noqa: E402 +from src import coverage_gate as cg # noqa: E402 + +_REPO = "orchestrator" +_BRANCH = "feature/ORCH-027-code-coverage" +_WI = "ORCH-027" + + +@pytest.fixture(autouse=True) +def fresh_db(tmp_path, monkeypatch): + """Isolated tmp SQLite DB + gate ON / empty scope (self-hosting) by default.""" + dbfile = tmp_path / "cov.db" + monkeypatch.setattr(db.settings, "db_path", str(dbfile)) + monkeypatch.setattr(cfg.settings, "coverage_gate_enabled", True, raising=False) + monkeypatch.setattr(cfg.settings, "coverage_gate_repos", "", raising=False) + monkeypatch.setattr(cfg.settings, "coverage_min_percent", 80.0, raising=False) + monkeypatch.setattr(cfg.settings, "coverage_policy", "both", raising=False) + monkeypatch.setattr(cfg.settings, "coverage_epsilon", 0.5, raising=False) + monkeypatch.setattr(cfg.settings, "coverage_tool_fail_closed", False, raising=False) + monkeypatch.setattr(cfg.settings, "coverage_run_timeout_s", 900, raising=False) + db.init_db() + yield + + +# =========================================================================== +# TC-01 — policy=absolute +# =========================================================================== +def test_tc01_policy_absolute(): + # measured >= floor -> PASS + ok, _ = cg.compute_coverage_verdict(85.0, None, 80.0, "absolute", 0.0) + assert ok is True + # exactly on the floor -> PASS (>=) + ok, _ = cg.compute_coverage_verdict(80.0, None, 80.0, "absolute", 0.0) + assert ok is True + # below floor-epsilon -> FAIL + ok, _ = cg.compute_coverage_verdict(78.0, None, 80.0, "absolute", 0.5) + assert ok is False + # baseline is IGNORED under absolute (even a high baseline cannot fail it) + ok, _ = cg.compute_coverage_verdict(85.0, 99.0, 80.0, "absolute", 0.0) + assert ok is True + + +# =========================================================================== +# TC-02 — policy=baseline (no-regression / ratchet) +# =========================================================================== +def test_tc02_policy_baseline(): + # measured >= baseline -> PASS + ok, _ = cg.compute_coverage_verdict(90.0, 85.0, 0.0, "baseline", 0.0) + assert ok is True + # exactly on baseline -> PASS + ok, _ = cg.compute_coverage_verdict(85.0, 85.0, 0.0, "baseline", 0.0) + assert ok is True + # below baseline-epsilon -> FAIL + ok, _ = cg.compute_coverage_verdict(83.0, 85.0, 0.0, "baseline", 0.5) + assert ok is False + # floor is IGNORED under baseline (low measured vs floor but >= baseline -> PASS) + ok, _ = cg.compute_coverage_verdict(40.0, 30.0, 80.0, "baseline", 0.0) + assert ok is True + # bootstrap: baseline None under baseline policy -> PASS (cannot regress vs nothing) + ok, reason = cg.compute_coverage_verdict(10.0, None, 80.0, "baseline", 0.0) + assert ok is True + assert "bootstrap" in reason.lower() + + +# =========================================================================== +# TC-03 — policy=both (PASS only if BOTH hold) +# =========================================================================== +def test_tc03_policy_both(): + # both hold -> PASS + ok, _ = cg.compute_coverage_verdict(90.0, 85.0, 80.0, "both", 0.0) + assert ok is True + # absolute fails (below floor) -> FAIL even though >= baseline + ok, _ = cg.compute_coverage_verdict(82.0, 80.0, 85.0, "both", 0.0) + assert ok is False + # baseline fails (below baseline) -> FAIL even though >= floor + ok, _ = cg.compute_coverage_verdict(84.0, 90.0, 80.0, "both", 0.0) + assert ok is False + # bootstrap under both: baseline None -> only absolute decides + ok, _ = cg.compute_coverage_verdict(85.0, None, 80.0, "both", 0.0) + assert ok is True + ok, _ = cg.compute_coverage_verdict(70.0, None, 80.0, "both", 0.0) + assert ok is False + + +# =========================================================================== +# TC-04 — epsilon tolerance (anti-flap, NFR-4) +# =========================================================================== +def test_tc04_epsilon_tolerance(): + # measured 0.3% under baseline, epsilon 0.5 -> still PASS (within noise) + ok, _ = cg.compute_coverage_verdict(84.7, 85.0, 80.0, "both", 0.5) + assert ok is True + # measured 0.3% under floor, epsilon 0.5 -> still PASS + ok, _ = cg.compute_coverage_verdict(79.7, 80.0, 0.0, "absolute", 0.5) + assert ok is True + # just beyond epsilon -> FAIL + ok, _ = cg.compute_coverage_verdict(84.4, 85.0, 80.0, "baseline", 0.5) + assert ok is False + # negative epsilon is clamped to 0 (no negative tolerance) + ok, _ = cg.compute_coverage_verdict(84.9, 85.0, 0.0, "baseline", -5.0) + assert ok is False + + +# =========================================================================== +# TC-05 — ratchet baseline (up only; never lowers) +# =========================================================================== +def test_tc05_ratchet_up_only(): + # bootstrap seeds the baseline + assert db.get_coverage_baseline(_REPO) is None + assert db.ratchet_coverage_baseline(_REPO, 80.0, "sha1") is True + assert db.get_coverage_baseline(_REPO) == pytest.approx(80.0) + # higher value raises it + assert db.ratchet_coverage_baseline(_REPO, 85.0, "sha2") is True + assert db.get_coverage_baseline(_REPO) == pytest.approx(85.0) + # equal value re-stamps (idempotent, no harm) — baseline unchanged + db.ratchet_coverage_baseline(_REPO, 85.0, "sha3") + assert db.get_coverage_baseline(_REPO) == pytest.approx(85.0) + # LOWER value does NOT lower the baseline + assert db.ratchet_coverage_baseline(_REPO, 70.0, "sha4") is False + assert db.get_coverage_baseline(_REPO) == pytest.approx(85.0) + + +def test_tc05_ratchet_per_repo_isolated(): + db.ratchet_coverage_baseline(_REPO, 85.0, "s") + db.ratchet_coverage_baseline("enduro-trails", 42.0, "s") + assert db.get_coverage_baseline(_REPO) == pytest.approx(85.0) + assert db.get_coverage_baseline("enduro-trails") == pytest.approx(42.0) + + +# =========================================================================== +# TC-06 — bootstrap baseline (first init from main measurement) +# =========================================================================== +def test_tc06_bootstrap(monkeypatch, tmp_path): + # No baseline yet -> ratchet_baseline_on_merge seeds it from the artefact value. + report = ( + "---\ncoverage_status: PASS\nwork_item: ORCH-027\n" + "measured_coverage: 77.50\nbaseline: \nfloor: 0.00\npolicy: both\n" + "epsilon: 0.50\ndelta: 0.00\n---\n# body\n" + ) + monkeypatch.setattr(cg, "_report_path", lambda *a, **k: str(tmp_path / "18.md")) + (tmp_path / "18.md").write_text(report, encoding="utf-8") + assert db.get_coverage_baseline(_REPO) is None + assert cg.ratchet_baseline_on_merge(_REPO, _WI, _BRANCH, "sha") is True + assert db.get_coverage_baseline(_REPO) == pytest.approx(77.5) + + +# =========================================================================== +# TC-07 — conditionality applies(repo) (empty scope -> self-hosting only) +# =========================================================================== +def test_tc07_applies_self_hosting_only(monkeypatch): + monkeypatch.setattr(cfg.settings, "coverage_gate_repos", "", raising=False) + assert cg.coverage_gate_applies("orchestrator") is True + assert cg.coverage_gate_applies("enduro-trails") is False + + +def test_tc07_applies_csv_scope(monkeypatch): + monkeypatch.setattr(cfg.settings, "coverage_gate_repos", "foo, enduro-trails", raising=False) + assert cg.coverage_gate_applies("enduro-trails") is True + assert cg.coverage_gate_applies("orchestrator") is False + + +def test_tc07_out_of_scope_noop_no_measure(monkeypatch): + # Out-of-scope repo -> (True, "...N/A") and the expensive measurer is NOT called. + called = {"n": 0} + monkeypatch.setattr(cg, "measure_coverage", lambda *a, **k: called.__setitem__("n", called["n"] + 1) or 99.0) + ok, reason = cg.check_coverage_gate("enduro-trails", "ET-1", "feature/x") + assert ok is True + assert "N/A" in reason + assert called["n"] == 0 + + +# =========================================================================== +# TC-08 — kill-switch off -> inert (1:1 as before ORCH-027) +# =========================================================================== +def test_tc08_kill_switch_off(monkeypatch): + monkeypatch.setattr(cfg.settings, "coverage_gate_enabled", False, raising=False) + called = {"n": 0} + monkeypatch.setattr(cg, "measure_coverage", lambda *a, **k: called.__setitem__("n", called["n"] + 1) or 10.0) + ok, reason = cg.check_coverage_gate(_REPO, _WI, _BRANCH) + assert ok is True + assert "disabled" in reason + assert called["n"] == 0 + assert cg.coverage_gate_applies(_REPO) is False + + +# =========================================================================== +# TC-09 — fail-open by default on a tool error; fail-closed when configured +# =========================================================================== +def test_tc09_fail_open_default(monkeypatch, tmp_path): + monkeypatch.setattr(cg, "measure_coverage", lambda *a, **k: None) # tool error + monkeypatch.setattr(cg, "_report_path", lambda *a, **k: str(tmp_path / "18.md")) + ok, reason = cg.check_coverage_gate(_REPO, _WI, _BRANCH) + assert ok is True + assert "fail-open" in reason.lower() + # The report records the fail-open PASS. + content = (tmp_path / "18.md").read_text(encoding="utf-8") + assert "coverage_status: PASS" in content + + +def test_tc09_fail_closed_when_configured(monkeypatch, tmp_path): + monkeypatch.setattr(cfg.settings, "coverage_tool_fail_closed", True, raising=False) + monkeypatch.setattr(cg, "measure_coverage", lambda *a, **k: None) + monkeypatch.setattr(cg, "_report_path", lambda *a, **k: str(tmp_path / "18.md")) + ok, reason = cg.check_coverage_gate(_REPO, _WI, _BRANCH) + assert ok is False + assert "fail-closed" in reason.lower() + content = (tmp_path / "18.md").read_text(encoding="utf-8") + assert "coverage_status: FAIL" in content + + +# =========================================================================== +# TC-10 — never-raise (broken inputs / internal error never escape) +# =========================================================================== +def test_tc10_verdict_never_raises_on_bad_inputs(): + ok, reason = cg.compute_coverage_verdict("not-a-number", None, 80.0, "both", 0.5) + assert ok is False + assert "bad inputs" in reason + + +def test_tc10_parse_coverage_percent_tolerant(): + assert cg.parse_coverage_percent({"totals": {"percent_covered": 73.2}}) == pytest.approx(73.2) + assert cg.parse_coverage_percent({}) is None + assert cg.parse_coverage_percent("garbage") is None + assert cg.parse_coverage_percent({"totals": {}}) is None + + +def test_tc10_check_never_raises(monkeypatch): + # measure_coverage explodes -> the gate swallows it and returns a pair (fail-open). + def _boom(*a, **k): + raise RuntimeError("coverage exploded") + monkeypatch.setattr(cg, "measure_coverage", _boom) + ok, reason = cg.check_coverage_gate(_REPO, _WI, _BRANCH) + assert isinstance(ok, bool) + assert "error (fail-open)" in reason + + +def test_tc10_ratchet_never_raises_on_missing_report(monkeypatch, tmp_path): + monkeypatch.setattr(cg, "_report_path", lambda *a, **k: str(tmp_path / "nope.md")) + assert cg.ratchet_baseline_on_merge(_REPO, _WI, _BRANCH, "sha") is False + + +# =========================================================================== +# TC-11 — write/read report; single source of truth via frontmatter +# =========================================================================== +def test_tc11_report_roundtrip(tmp_path): + fields = { + "coverage_status": "PASS", + "measured_coverage": 88.25, + "baseline": 85.0, + "floor": 80.0, + "policy": "both", + "epsilon": 0.5, + "delta": 3.25, + "reason": "ok", + "measurement": "pytest --cov=src: 88.25%", + "policy_detail": "policy=both", + } + content = cg.render_coverage_report(_WI, fields) + # machine key present and parseable + ok, verdict = cg.parse_coverage_status(content) + assert ok is True + assert "PASS" in verdict + # measured_coverage read back from the SAME file (ratchet source of truth) + assert cg.read_measured_coverage(content) == pytest.approx(88.25) + + # FAIL roundtrip (FAIL token authoritative) + fields["coverage_status"] = "FAIL" + content = cg.render_coverage_report(_WI, fields) + ok, verdict = cg.parse_coverage_status(content) + assert ok is False + assert "FAIL" in verdict + + +def test_tc11_parse_missing_frontmatter(): + ok, reason = cg.parse_coverage_status("no frontmatter here") + assert ok is False + assert "coverage_status" in reason + assert cg.read_measured_coverage("no frontmatter") is None + + +def test_tc11_bootstrap_report_blank_baseline(): + # bootstrap: baseline None -> renders an EMPTY baseline field, still parseable. + fields = { + "coverage_status": "PASS", "measured_coverage": 50.0, "baseline": None, + "floor": 0.0, "policy": "both", "epsilon": 0.5, "delta": 0.0, + } + content = cg.render_coverage_report(_WI, fields) + assert "baseline: \n" in content or "baseline:\n" in content + assert cg.parse_coverage_status(content)[0] is True + + +# =========================================================================== +# TC-12 — self-hosting safety: the leaf imports no engine, touches no prod +# =========================================================================== +def test_tc12_leaf_no_engine_import(): + # AST-based (not prose): the leaf must never IMPORT the engine, and the only + # external command it runs is pytest — no docker/compose/force-push literals. + import ast + import inspect + tree = ast.parse(inspect.getsource(cg)) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + elif isinstance(node, ast.Import): + for n in node.names: + imported.add(n.name) + assert not any("stage_engine" in m for m in imported), imported + assert not any(("launcher" in m or "self_deploy" in m) for m in imported), imported + # No deploy / restart / force-push command tokens used as actual string literals. + consts = [ + n.value for n in ast.walk(tree) + if isinstance(n, ast.Constant) and isinstance(n.value, str) + ] + for forbidden in ("compose", "--force-with-lease", "--force", "docker"): + assert forbidden not in consts, f"coverage_gate leaf must not run {forbidden!r}" + + +def test_tc12_delta_signed(): + assert cg.compute_delta(85.0, 80.0, 70.0) == pytest.approx(5.0) # vs max(80,70) + assert cg.compute_delta(75.0, 80.0, 70.0) == pytest.approx(-5.0) + assert cg.compute_delta(50.0, None, None) == pytest.approx(0.0) + + +# =========================================================================== +# TC-13 — gate integration into advance_stage (rollback on FAIL, retry++) +# =========================================================================== +def test_tc13_advance_rolls_back_on_fail(monkeypatch): + from src import stage_engine as se + + captured = {} + + def _fake_run_qg(name, repo, wi, branch): + captured["qg"] = name + return (False, "measured=70.00% policy=both: absolute FAIL") + + monkeypatch.setattr(se, "_run_qg", _fake_run_qg) + monkeypatch.setattr(se, "update_task_stage", lambda *a, **k: None) + monkeypatch.setattr(se, "notify_stage_change", lambda *a, **k: None) + monkeypatch.setattr(se, "plane_notify_stage", lambda *a, **k: None) + monkeypatch.setattr(se, "set_issue_in_progress", lambda *a, **k: None) + monkeypatch.setattr(se, "notify_qg_failure", lambda *a, **k: None) + monkeypatch.setattr(se, "plane_add_comment", lambda *a, **k: None) + monkeypatch.setattr(se, "_developer_retry_count", lambda *a, **k: 0) + released = {"n": 0} + monkeypatch.setattr(se.merge_gate, "release_merge_lease", + lambda *a, **k: released.__setitem__("n", released["n"] + 1)) + enq = {"n": 0} + monkeypatch.setattr(se, "enqueue_job", + lambda *a, **k: enq.__setitem__("n", enq["n"] + 1) or 123) + + result = se.AdvanceResult() + intervened = se._handle_coverage_gate(1, "deploy-staging", _REPO, _WI, _BRANCH, "deployer", result) + assert intervened is True + assert captured["qg"] == "check_coverage_gate" + assert result.rolled_back_to == "development" + assert result.enqueued_agent == "developer" + assert enq["n"] == 1 + # merge lease released on the coverage rollback (ADR-001 D1/TR-2) + assert released["n"] == 1 + + +def test_tc13_advance_passes_through_on_ok(monkeypatch): + from src import stage_engine as se + monkeypatch.setattr(se, "_run_qg", lambda *a, **k: (True, "coverage OK")) + result = se.AdvanceResult() + intervened = se._handle_coverage_gate(1, "deploy-staging", _REPO, _WI, _BRANCH, "deployer", result) + assert intervened is False + assert result.rolled_back_to is None + + +# =========================================================================== +# TC-14 — real measurement on a minimal fixture repo (pytest --cov in worktree) +# =========================================================================== +def test_tc14_real_measurement(tmp_path, monkeypatch): + # Build a minimal project: src/ with one function, tests covering part of it. + proj = tmp_path / "fixture_repo" + (proj / "src").mkdir(parents=True) + (proj / "tests").mkdir() + (proj / "src" / "__init__.py").write_text("", encoding="utf-8") + (proj / "src" / "mod.py").write_text( + "def covered():\n return 1\n\n\ndef uncovered():\n return 2\n", + encoding="utf-8", + ) + (proj / "tests" / "test_mod.py").write_text( + "from src.mod import covered\n\n\ndef test_covered():\n assert covered() == 1\n", + encoding="utf-8", + ) + # Point the measurer's worktree resolution at our fixture. + monkeypatch.setattr(cg, "ensure_worktree", lambda repo, branch: str(proj)) + pct = cg.measure_coverage(_REPO, _BRANCH) + assert pct is not None + # mod.py: 4 statements, uncovered() body (1) unrun -> ~75%; bounds-check only. + assert 50.0 <= pct <= 90.0 + # the scratch json is cleaned up + assert not (proj / ".coverage-report.json").exists() + + +def test_tc14_measure_timeout_returns_none(monkeypatch): + import subprocess + monkeypatch.setattr(cg, "ensure_worktree", lambda r, b: "/tmp") + + def _timeout(*a, **k): + raise subprocess.TimeoutExpired(cmd="pytest", timeout=1) + monkeypatch.setattr(cg.subprocess, "run", _timeout) + assert cg.measure_coverage(_REPO, _BRANCH) is None + + +# =========================================================================== +# TC-15 — observability (snapshot block) + registry compatibility unchanged +# =========================================================================== +def test_tc15_snapshot_shape(monkeypatch): + db.ratchet_coverage_baseline(_REPO, 81.0, "sha") + snap = cg.snapshot() + assert snap["enabled"] is True + assert snap["policy"] == "both" + assert snap["floor"] == pytest.approx(80.0) + assert "baselines" in snap + assert _REPO in snap["baselines"] + assert snap["baselines"][_REPO]["coverage"] == pytest.approx(81.0) + + +def test_tc15_snapshot_never_raises(monkeypatch): + monkeypatch.setattr(db, "all_coverage_baselines", lambda: (_ for _ in ()).throw(RuntimeError("boom"))) + snap = cg.snapshot() + assert snap["enabled"] is True + assert snap["baselines"] == {} + + +def test_tc15_registry_and_transitions_unchanged(): + from src.qg.checks import QG_CHECKS + from src.stages import STAGE_TRANSITIONS + # new check registered... + assert "check_coverage_gate" in QG_CHECKS + # ...without touching the existing verdict checks (byte-for-byte names present) + for name in ( + "check_ci_green", "check_tests_passed", "check_security_gate", + "check_staging_status", "check_staging_image_fresh", "check_branch_mergeable", + ): + assert name in QG_CHECKS + # coverage is an edge sub-gate, NOT a STAGE_TRANSITIONS edge + for _stage, spec in STAGE_TRANSITIONS.items(): + assert "check_coverage_gate" not in str(spec) diff --git a/tests/test_plane_status_model.py b/tests/test_plane_status_model.py index 33edd00..a330573 100644 --- a/tests/test_plane_status_model.py +++ b/tests/test_plane_status_model.py @@ -141,6 +141,7 @@ def test_tc23_qg_checks_registry_unchanged(): "check_reviewer_verdict", "check_tests_local", "check_deploy_status", "check_staging_status", "check_branch_mergeable", "check_staging_image_fresh", "check_security_gate", # ORCH-022 integ: security-gate registered + "check_coverage_gate", # ORCH-027 integ: coverage-gate registered } diff --git a/tests/test_qg_registry_snapshot.py b/tests/test_qg_registry_snapshot.py index 1c8c44a..5111d8e 100644 --- a/tests/test_qg_registry_snapshot.py +++ b/tests/test_qg_registry_snapshot.py @@ -31,6 +31,7 @@ _EXPECTED_QGS = { "check_branch_mergeable", # ORCH-043 merge-gate (deploy-staging -> deploy edge) "check_staging_image_fresh", # ORCH-058 image-freshness sub-gate (same edge) "check_security_gate", # ORCH-022 security sub-gate (same edge, run FIRST) + "check_coverage_gate", # ORCH-027 coverage sub-gate (same edge, after merge-gate) } diff --git a/tests/test_stage_engine.py b/tests/test_stage_engine.py index 66ced68..2926f1a 100644 --- a/tests/test_stage_engine.py +++ b/tests/test_stage_engine.py @@ -833,6 +833,7 @@ class TestMergeGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _pass, "check_staging_image_fresh": _pass}, ) @@ -858,6 +859,7 @@ class TestMergeGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _fail("merge-lock busy")}, ) monkeypatch.setattr(stage_engine.settings, "merge_defer_delay_s", 30) @@ -886,6 +888,7 @@ class TestMergeGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _fail("merge-lock busy")}, ) monkeypatch.setattr(stage_engine.settings, "merge_defer_max_attempts", 3) @@ -920,6 +923,7 @@ class TestMergeGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _fail("rebase conflict: src/db.py")}, ) task_id = _make_task("deploy-staging", repo="orchestrator", wi="ORCH-043", @@ -944,6 +948,7 @@ class TestMergeGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _fail("re-test failed after rebase: 1 failed")}, ) task_id = _make_task("deploy-staging", repo="orchestrator", wi="ORCH-043", @@ -968,6 +973,7 @@ class TestMergeGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _fail("rebase conflict: src/db.py")}, ) task_id = _make_task("deploy-staging", repo="orchestrator", wi="ORCH-043", @@ -1021,6 +1027,7 @@ class TestImageFreshnessGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _pass, "check_staging_image_fresh": _fail( "staging rebuild failed: health FAILED")}, @@ -1049,6 +1056,7 @@ class TestImageFreshnessGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _pass, "check_staging_image_fresh": _fail("provenance mismatch")}, ) @@ -1073,6 +1081,7 @@ class TestImageFreshnessGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _pass, "check_staging_image_fresh": _pass}, ) @@ -1099,6 +1108,7 @@ class TestImageFreshnessGate: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _pass}, ) # check_staging_image_fresh left REAL -> N/A for enduro-trails task_id = _make_task("deploy-staging", repo="enduro-trails", wi="ET-099", @@ -1171,6 +1181,7 @@ class TestStagingInfraTolerance: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _pass, "check_staging_image_fresh": _pass}, ) @@ -1244,6 +1255,7 @@ class TestStagingInfraTolerance: {**stage_engine.QG_CHECKS, "check_staging_status": _pass, "check_security_gate": _pass, + "check_coverage_gate": _pass, "check_branch_mergeable": _pass, "check_staging_image_fresh": _pass, "check_deploy_status": _pass}, diff --git a/tests/test_stages_invariants.py b/tests/test_stages_invariants.py index 171581a..5714cb2 100644 --- a/tests/test_stages_invariants.py +++ b/tests/test_stages_invariants.py @@ -27,6 +27,7 @@ _EXPECTED_QGS = { "check_branch_mergeable", "check_staging_image_fresh", "check_security_gate", + "check_coverage_gate", } _EXPECTED_TRANSITIONS = {