claude-bot
4bebb921ff
Closes the "zombie jobs" incident class: job status was set only inside the live launcher process, so a process death left jobs.status='running' forever; at max_concurrency=1 one zombie blocked ALL projects' queue (self-hosting risk). Adds a background daemon (src/job_reaper.py) with three-tier liveness (dead-pid streak / known exit_code / max-running backstop) whose only mutating write is an atomic terminal flip guarded by WHERE status='running' (no double-process). For exit0 the canonical QG is the source of truth via gate-driven advance, not "exit0". Also proactively reclaims stale merge-lease (dead pid OR TTL) via file delete only (no git ops), and makes merge finalization idempotent (pr_already_merged guard + up-to-date short-circuit on re-drive). New jobs.pid column via idempotent _ensure_column (no migration); pid stamped in launcher._spawn after Popen. Reaper start/stop in lifespan; "reaper" snapshot in GET /queue. Kill-switches: ORCH_REAPER_ENABLED, ORCH_REAPER_INTERVAL_S, ORCH_REAPER_DEAD_TICKS, ORCH_REAPER_MAX_RUNNING_S, ORCH_LEASE_RECLAIM_ENABLED. Invariants unchanged (AC-13): STAGE_TRANSITIONS, QG_CHECKS registry, check_branch_mergeable signature/behaviour, BUG-8 rollback, hook exit codes. restart-safe, never-raise per unit of background work. Docs: docs/architecture/README.md, CHANGELOG.md, .env.example. Tests: tests/test_job_reaper.py, tests/test_merge_lease_reclaim.py, tests/test_merge_gate.py (TC-16), tests/test_merge_gate_race.py (TC-17), tests/test_queue.py, tests/test_config.py (TC-19/TC-20). 742 passed. Refs: ORCH-065 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
247 lines
9.7 KiB
Python
247 lines
9.7 KiB
Python
"""ORCH-042: Settings.tracker_mode config field.
|
|
|
|
AC-1: tracker_mode defaults to "edit" and is read from env ORCH_TRACKER_MODE.
|
|
Settings is a Pydantic BaseSettings reading env at instantiation, so each case
|
|
builds a FRESH Settings() (the process-wide singleton is not mutated).
|
|
"""
|
|
|
|
from src.config import Settings
|
|
|
|
|
|
def test_tracker_mode_defaults_to_edit(monkeypatch):
|
|
# No env var -> default "edit" (TC-01 / AC-1).
|
|
monkeypatch.delenv("ORCH_TRACKER_MODE", raising=False)
|
|
assert Settings().tracker_mode == "edit"
|
|
|
|
|
|
def test_tracker_mode_reads_env_bump(monkeypatch):
|
|
# ORCH_TRACKER_MODE=bump -> "bump" (TC-01 / AC-1).
|
|
monkeypatch.setenv("ORCH_TRACKER_MODE", "bump")
|
|
assert Settings().tracker_mode == "bump"
|
|
|
|
|
|
def test_tracker_mode_reads_env_arbitrary(monkeypatch):
|
|
# The field is read verbatim from env; mode RESOLUTION (anything != "bump"
|
|
# -> edit) happens in notifications, not here (AC-1/AC-2 split).
|
|
monkeypatch.setenv("ORCH_TRACKER_MODE", "garbage")
|
|
assert Settings().tracker_mode == "garbage"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ORCH-043 / TC-25: merge-gate settings defaults + env override.
|
|
# ---------------------------------------------------------------------------
|
|
_MERGE_ENV = (
|
|
"ORCH_MERGE_GATE_ENABLED",
|
|
"ORCH_MERGE_GATE_REPOS",
|
|
"ORCH_MERGE_RETEST_TIMEOUT_S",
|
|
"ORCH_MERGE_RETEST_TARGET",
|
|
"ORCH_MERGE_LOCK_TIMEOUT_S",
|
|
"ORCH_MERGE_DEFER_DELAY_S",
|
|
"ORCH_MERGE_DEFER_MAX_ATTEMPTS",
|
|
)
|
|
|
|
|
|
def test_merge_gate_settings_defaults(monkeypatch):
|
|
"""TC-25 / AC-10: documented defaults when no env is set."""
|
|
for name in _MERGE_ENV:
|
|
monkeypatch.delenv(name, raising=False)
|
|
s = Settings()
|
|
assert s.merge_gate_enabled is True
|
|
assert s.merge_gate_repos == ""
|
|
assert s.merge_retest_timeout_s == 600
|
|
assert s.merge_retest_target == "tests/"
|
|
assert s.merge_lock_timeout_s == 300
|
|
assert s.merge_defer_delay_s == 60
|
|
assert s.merge_defer_max_attempts == 5
|
|
|
|
|
|
def test_merge_gate_settings_env_override(monkeypatch):
|
|
"""TC-25 / AC-10: each field is read from its ORCH_* env var."""
|
|
monkeypatch.setenv("ORCH_MERGE_GATE_ENABLED", "false")
|
|
monkeypatch.setenv("ORCH_MERGE_GATE_REPOS", "orchestrator,enduro-trails")
|
|
monkeypatch.setenv("ORCH_MERGE_RETEST_TIMEOUT_S", "120")
|
|
monkeypatch.setenv("ORCH_MERGE_RETEST_TARGET", "tests/unit")
|
|
monkeypatch.setenv("ORCH_MERGE_LOCK_TIMEOUT_S", "90")
|
|
monkeypatch.setenv("ORCH_MERGE_DEFER_DELAY_S", "5")
|
|
monkeypatch.setenv("ORCH_MERGE_DEFER_MAX_ATTEMPTS", "9")
|
|
s = Settings()
|
|
assert s.merge_gate_enabled is False
|
|
assert s.merge_gate_repos == "orchestrator,enduro-trails"
|
|
assert s.merge_retest_timeout_s == 120
|
|
assert s.merge_retest_target == "tests/unit"
|
|
assert s.merge_lock_timeout_s == 90
|
|
assert s.merge_defer_delay_s == 5
|
|
assert s.merge_defer_max_attempts == 9
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ORCH-053 / TC-22: reconcile_* settings defaults + env override.
|
|
# ---------------------------------------------------------------------------
|
|
_RECONCILE_ENV = (
|
|
"ORCH_RECONCILE_ENABLED",
|
|
"ORCH_RECONCILE_INTERVAL_S",
|
|
"ORCH_RECONCILE_PLANE_ENABLED",
|
|
"ORCH_RECONCILE_GRACE_DEFAULT_S",
|
|
"ORCH_RECONCILE_GRACE_OVERRIDES_JSON",
|
|
"ORCH_RECONCILE_NOTIFY_UNBLOCK",
|
|
)
|
|
|
|
|
|
def test_reconcile_settings_defaults(monkeypatch):
|
|
"""TC-22 / AC-13: documented defaults when no env is set."""
|
|
for name in _RECONCILE_ENV:
|
|
monkeypatch.delenv(name, raising=False)
|
|
s = Settings()
|
|
assert s.reconcile_enabled is True
|
|
assert s.reconcile_interval_s == 120
|
|
assert s.reconcile_plane_enabled is True
|
|
assert s.reconcile_grace_default_s == 600
|
|
assert s.reconcile_grace_overrides_json == ""
|
|
assert s.reconcile_notify_unblock is True
|
|
|
|
|
|
def test_reconcile_settings_env_override(monkeypatch):
|
|
"""TC-22 / AC-13: each field is read from its ORCH_* env var."""
|
|
monkeypatch.setenv("ORCH_RECONCILE_ENABLED", "false")
|
|
monkeypatch.setenv("ORCH_RECONCILE_INTERVAL_S", "300")
|
|
monkeypatch.setenv("ORCH_RECONCILE_PLANE_ENABLED", "false")
|
|
monkeypatch.setenv("ORCH_RECONCILE_GRACE_DEFAULT_S", "900")
|
|
monkeypatch.setenv("ORCH_RECONCILE_GRACE_OVERRIDES_JSON", '{"development": 300}')
|
|
monkeypatch.setenv("ORCH_RECONCILE_NOTIFY_UNBLOCK", "false")
|
|
s = Settings()
|
|
assert s.reconcile_enabled is False
|
|
assert s.reconcile_interval_s == 300
|
|
assert s.reconcile_plane_enabled is False
|
|
assert s.reconcile_grace_default_s == 900
|
|
assert s.reconcile_grace_overrides_json == '{"development": 300}'
|
|
assert s.reconcile_notify_unblock is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ORCH-058 / TC-13: image-freshness settings defaults + env override.
|
|
# ---------------------------------------------------------------------------
|
|
_FRESH_ENV = (
|
|
"ORCH_IMAGE_FRESHNESS_ENABLED",
|
|
"ORCH_IMAGE_FRESHNESS_REPOS",
|
|
)
|
|
|
|
|
|
def test_image_freshness_settings_defaults(monkeypatch):
|
|
"""TC-13 / AC-9: kill-switch ON by default, empty CSV (self-hosting only)."""
|
|
for name in _FRESH_ENV:
|
|
monkeypatch.delenv(name, raising=False)
|
|
s = Settings()
|
|
assert s.image_freshness_enabled is True
|
|
assert s.image_freshness_repos == ""
|
|
|
|
|
|
def test_image_freshness_settings_env_override(monkeypatch):
|
|
"""TC-13 / AC-9: each field is read from its ORCH_* env var."""
|
|
monkeypatch.setenv("ORCH_IMAGE_FRESHNESS_ENABLED", "false")
|
|
monkeypatch.setenv("ORCH_IMAGE_FRESHNESS_REPOS", "orchestrator,enduro-trails")
|
|
s = Settings()
|
|
assert s.image_freshness_enabled is False
|
|
assert s.image_freshness_repos == "orchestrator,enduro-trails"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ORCH-061 / TC-09: staging_infra_tolerance_enabled kill-switch (AC-7).
|
|
# ---------------------------------------------------------------------------
|
|
def test_staging_infra_tolerance_defaults_true(monkeypatch):
|
|
"""TC-09 / AC-7: the kill-switch defaults ON (safe default — the safety net
|
|
holds regardless; the flag exists to restore legacy strictness instantly)."""
|
|
monkeypatch.delenv("ORCH_STAGING_INFRA_TOLERANCE_ENABLED", raising=False)
|
|
assert Settings().staging_infra_tolerance_enabled is True
|
|
|
|
|
|
def test_staging_infra_tolerance_env_override_false(monkeypatch):
|
|
"""TC-09 / AC-7: ORCH_STAGING_INFRA_TOLERANCE_ENABLED=false -> strict (1:1
|
|
pre-ORCH-061: infra-only FAIL again rolls back)."""
|
|
monkeypatch.setenv("ORCH_STAGING_INFRA_TOLERANCE_ENABLED", "false")
|
|
assert Settings().staging_infra_tolerance_enabled is False
|
|
|
|
|
|
def test_staging_infra_tolerance_env_override_true(monkeypatch):
|
|
"""The field is read verbatim from its ORCH_* env var."""
|
|
monkeypatch.setenv("ORCH_STAGING_INFRA_TOLERANCE_ENABLED", "true")
|
|
assert Settings().staging_infra_tolerance_enabled is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ORCH-065 / TC-20: reaper_* + lease_reclaim_* settings defaults + env override.
|
|
# ---------------------------------------------------------------------------
|
|
_REAPER_ENV = (
|
|
"ORCH_REAPER_ENABLED",
|
|
"ORCH_REAPER_INTERVAL_S",
|
|
"ORCH_REAPER_DEAD_TICKS",
|
|
"ORCH_REAPER_MAX_RUNNING_S",
|
|
"ORCH_LEASE_RECLAIM_ENABLED",
|
|
)
|
|
|
|
|
|
def test_reaper_settings_defaults(monkeypatch):
|
|
"""TC-20 / §5: documented defaults when no env is set."""
|
|
for name in _REAPER_ENV:
|
|
monkeypatch.delenv(name, raising=False)
|
|
s = Settings()
|
|
assert s.reaper_enabled is True
|
|
assert s.reaper_interval_s == 60
|
|
assert s.reaper_dead_ticks == 2
|
|
assert s.reaper_max_running_s == 3600
|
|
assert s.lease_reclaim_enabled is True
|
|
|
|
|
|
def test_reaper_settings_env_override(monkeypatch):
|
|
"""TC-20 / §5 / AC-14: each field is read from its ORCH_* env var."""
|
|
monkeypatch.setenv("ORCH_REAPER_ENABLED", "false")
|
|
monkeypatch.setenv("ORCH_REAPER_INTERVAL_S", "30")
|
|
monkeypatch.setenv("ORCH_REAPER_DEAD_TICKS", "5")
|
|
monkeypatch.setenv("ORCH_REAPER_MAX_RUNNING_S", "1200")
|
|
monkeypatch.setenv("ORCH_LEASE_RECLAIM_ENABLED", "false")
|
|
s = Settings()
|
|
assert s.reaper_enabled is False
|
|
assert s.reaper_interval_s == 30
|
|
assert s.reaper_dead_ticks == 5
|
|
assert s.reaper_max_running_s == 1200
|
|
assert s.lease_reclaim_enabled is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ORCH-065 / TC-19: contracts unchanged — no new stages / QG checks; the
|
|
# check_branch_mergeable signature is intact (AC-13).
|
|
# ---------------------------------------------------------------------------
|
|
def test_tc19_stage_transitions_unchanged():
|
|
"""No new pipeline stage was introduced by ORCH-065."""
|
|
from src.stages import STAGE_TRANSITIONS
|
|
assert set(STAGE_TRANSITIONS) == {
|
|
"created", "analysis", "architecture", "development", "review",
|
|
"testing", "deploy-staging", "deploy", "done",
|
|
}
|
|
|
|
|
|
def test_tc19_qg_checks_registry_unchanged():
|
|
"""No new quality-gate check was added to the registry by ORCH-065."""
|
|
from src.qg.checks import QG_CHECKS
|
|
assert set(QG_CHECKS) == {
|
|
"check_analysis_approved",
|
|
"check_analysis_complete",
|
|
"check_architecture_done",
|
|
"check_ci_green",
|
|
"check_review_approved",
|
|
"check_tests_passed",
|
|
"check_reviewer_verdict",
|
|
"check_tests_local",
|
|
"check_deploy_status",
|
|
"check_staging_status",
|
|
"check_branch_mergeable",
|
|
"check_staging_image_fresh",
|
|
}
|
|
|
|
|
|
def test_tc19_check_branch_mergeable_signature_intact():
|
|
"""check_branch_mergeable still takes exactly (repo, work_item_id, branch)."""
|
|
import inspect
|
|
from src.qg.checks import check_branch_mergeable
|
|
params = list(inspect.signature(check_branch_mergeable).parameters)
|
|
assert params == ["repo", "work_item_id", "branch"]
|