orchestrator/tests/test_deploy_checkout_hygiene.py

"""ORCH-112: deploy-base checkout-hygiene (resilient-pull) — TC-01…TC-10.

Two test layers:

  * SHELL simulation (TC-01..TC-04, TC-07) — drives the REAL
    ``scripts/orchestrator-deploy-hook.sh`` in a hermetic sandbox. ``git`` is REAL
    (against a LOCAL bare "origin" — no network), while ``docker`` / ``curl`` /
    ``sleep`` are PATH-shimmed stubs so no real infra is touched and prod is never
    restarted (INFRA safety). Models tests/test_deploy_hook_rollback_sim.py.

  * UNIT (TC-05, TC-06, TC-08, TC-09, TC-10) — the ``checkout_hygiene`` leaf, the
    static safety contract of the hook (never ``-x`` / explicit excludes), the
    pipeline-invariant guard and the documentation invariant.

TC-01 is the MANDATORY incident-reproduction regression (ORCH-111): a dirty tracked
edit to ``src/config.py`` over an advanced ``origin/main`` makes the bare ``git pull``
abort with "local changes would be overwritten by merge" (RED before the fix); the
resilient-pull hygiene converges the base and the deploy proceeds (GREEN after).
"""

import os
import shutil
import stat
import subprocess

import pytest

from src import checkout_hygiene

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
HOOK = os.path.join(ROOT, "scripts", "orchestrator-deploy-hook.sh")

pytestmark = pytest.mark.skipif(
    shutil.which("bash") is None or shutil.which("git") is None,
    reason="bash + git required for the deploy-hook hygiene simulation",
)

# Distinctive file bodies so assertions prove WHICH version won.
_V1 = "ORIGIN-V1\n"
_V2 = "ORIGIN-V2-ADVANCED\n"
_DIRTY = "DIRTY-LOCAL-WIP\n"

# Isolate git from any host/global config (hermetic).
_GIT_ENV = {
    "GIT_AUTHOR_NAME": "t",
    "GIT_AUTHOR_EMAIL": "t@example.com",
    "GIT_COMMITTER_NAME": "t",
    "GIT_COMMITTER_EMAIL": "t@example.com",
    "GIT_CONFIG_GLOBAL": os.devnull,
    "GIT_CONFIG_SYSTEM": os.devnull,
}


def _git(cwd, *args):
    r = subprocess.run(
        ["git", "-C", str(cwd), *args],
        capture_output=True, text=True, env={**os.environ, **_GIT_ENV},
    )
    assert r.returncode == 0, f"git {args} failed: {r.stdout}\n{r.stderr}"
    return r


def _write(path, content):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "w", encoding="utf-8") as f:
        f.write(content)


def _write_exec(path, content):
    _write(path, content)
    os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)


def _make_stubs(binx, prev_running=True):
    """Healthy docker/curl/sleep stubs (no real infra; deploy always succeeds).

    ``prev_running`` controls whether ``docker compose ps -q`` returns a container id:
    True -> step 1 captures a previous image and writes PREV_IMAGE_FILE (the normal
    case); False -> no previous image is recorded (first-deploy / service-down), so the
    deploy-base stays genuinely clean at the hygiene step (exercises the no-op branch).
    """
    ps_id = "fakecid" if prev_running else ""
    _write_exec(str(binx / "docker"), f"""#!/bin/bash
case "$1" in
  compose)
    for a in "$@"; do [ "$a" = "ps" ] && {{ echo "{ps_id}"; exit 0; }}; done
    exit 0;;
  inspect) echo "sha256:previmage"; exit 0;;
  image) exit 0;;
  tag) exit 0;;
  *) exit 0;;
esac
""")
    # curl: ALWAYS healthy -> deploy health-check passes immediately -> exit 0.
    _write_exec(str(binx / "curl"), """#!/bin/bash
iscode=""
for a in "$@"; do [ "$a" = "-w" ] && iscode=1; done
[ -n "$iscode" ] && echo "200" || echo '{"status":"ok"}'
exit 0
""")
    _write_exec(str(binx / "sleep"), "#!/bin/bash\nexit 0\n")


def _seed_origin_and_clone(tmp_path):
    """Build a local bare origin (at V2) + a deploy-base clone (at V1).

    Returns ``repo`` (the deploy-base path). The clone is one commit BEHIND origin so
    that, with a conflicting dirty edit to src/config.py, a bare ``git pull`` would
    abort (the exact ORCH-111 incident), while hygiene's reset --hard converges it.
    """
    work = tmp_path / "work"
    work.mkdir()
    _write(str(work / "src" / "config.py"), _V1)
    _write(str(work / ".gitignore"), ".env\ndata/\n*.db\nbuild/\n")
    _git(work, "init", "-q")
    _git(work, "add", "-A")
    _git(work, "commit", "-q", "-m", "v1")
    _git(work, "branch", "-M", "main")

    origin = tmp_path / "origin.git"
    _git(tmp_path, "init", "-q", "--bare", str(origin))
    _git(work, "remote", "add", "origin", str(origin))
    _git(work, "push", "-q", "-u", "origin", "main")

    repo = tmp_path / "repo"
    _git(tmp_path, "clone", "-q", str(origin), str(repo))

    # Advance origin/main to V2 (touches the SAME file we will dirty locally).
    _write(str(work / "src" / "config.py"), _V2)
    _git(work, "commit", "-q", "-am", "v2")
    _git(work, "push", "-q", "origin", "main")
    # Make repo's remote-tracking ref aware of V2's existence is the hook's job
    # (it runs `git fetch`); leave repo at V1 deliberately.
    return repo


def _run_hook(repo, tmp_path, hygiene="1", extra_env=None, prev_running=True):
    """Run the real hook in --deploy mode against ``repo`` with stubbed infra."""
    binx = tmp_path / "bin"
    if not binx.exists():
        binx.mkdir()
        _make_stubs(binx, prev_running=prev_running)
    state = tmp_path / "state"
    state.mkdir(exist_ok=True)
    env = {
        **os.environ,
        **_GIT_ENV,
        "PATH": f"{binx}:{os.environ['PATH']}",
        "REPO": str(repo),
        "LOG": str(state / "hook.log"),
        "PREV_IMAGE_FILE": str(repo / ".deploy-prev-image-prod"),
        "COMPOSE_PROFILE": "",
        "TARGET_SERVICE": "orchestrator",
        "TARGET_PORT": "8500",
    }
    if hygiene is not None:
        env["CHECKOUT_HYGIENE"] = hygiene
        env["HYGIENE_REPORT"] = str(state / "hygiene")
    if extra_env:
        env.update(extra_env)
    return subprocess.run(
        ["bash", HOOK, "--deploy"], env=env, capture_output=True, text=True, timeout=60,
    )


def _porcelain(repo):
    r = subprocess.run(
        ["git", "-C", str(repo), "status", "--porcelain"],
        capture_output=True, text=True, env={**os.environ, **_GIT_ENV},
    )
    return r.stdout.strip()


def _wip_dirt(repo):
    """Porcelain output MINUS the intentionally-preserved deploy artefacts.

    After hygiene, the deploy-base is converged to origin/main but the rollback/log
    artefacts (.deploy-prev-image-* / deploy-hook.log) are legitimately untracked-and-
    preserved (NFR-2). This returns ONLY the *real* residual dirt (a non-empty result
    means a tracked edit survived or WIP was not cleaned)."""
    lines = []
    for ln in _porcelain(repo).splitlines():
        name = ln[3:] if len(ln) > 3 else ""
        if name.startswith(".deploy-prev-image-") or name == "deploy-hook.log":
            continue
        lines.append(ln)
    return "\n".join(lines).strip()


def _head_config(repo):
    with open(repo / "src" / "config.py", encoding="utf-8") as f:
        return f.read()


# ===========================================================================
# TC-01 — MANDATORY regression (red->green): dirty tracked edit + advanced origin
# ===========================================================================
def test_tc01_dirty_tracked_edit_converges_and_deploys(tmp_path):
    repo = _seed_origin_and_clone(tmp_path)
    # Dirty the SAME tracked file origin advanced -> a bare `git pull` would abort.
    _write(str(repo / "src" / "config.py"), _DIRTY)
    # Untracked WIP left behind too (failed/cancelled task residue).
    _write(str(repo / "scripts" / "install_lite.py"), "# wip\n")

    proc = _run_hook(repo, tmp_path, hygiene="1")

    assert proc.returncode == 0, (
        "resilient-pull must converge a dirty base and let the deploy proceed "
        f"(stdout={proc.stdout}\nstderr={proc.stderr})"
    )
    # Base converged to the ADVANCED origin/main (dirty local edit discarded).
    assert _head_config(repo) == _V2
    # No real WIP remains (tracked edit reset, untracked WIP cleaned); the rollback
    # snapshot .deploy-prev-image-prod is legitimately preserved (NFR-2), so we check
    # the residual dirt MINUS the preserved artefacts.
    assert _wip_dirt(repo) == ""
    assert not (repo / "scripts" / "install_lite.py").exists()
    out = proc.stdout + proc.stderr
    assert "HYGIENE" in out


def test_tc01b_bare_pull_aborts_without_hygiene_documents_incident(tmp_path):
    """ORCH-111 reproduction: WITHOUT hygiene the same dirty base aborts the pull."""
    repo = _seed_origin_and_clone(tmp_path)
    _write(str(repo / "src" / "config.py"), _DIRTY)

    proc = _run_hook(repo, tmp_path, hygiene=None)  # CHECKOUT_HYGIENE unset

    assert proc.returncode != 0, "bare `git pull` must abort on the conflicting dirty edit"
    log = (tmp_path / "state" / "hook.log").read_text(encoding="utf-8")
    assert "would be overwritten by merge" in (log + proc.stdout + proc.stderr)


# ===========================================================================
# TC-02 — untracked WIP files do not block and do not leak into the deploy
# ===========================================================================
def test_tc02_untracked_wip_does_not_block(tmp_path):
    repo = _seed_origin_and_clone(tmp_path)
    for rel in (
        "scripts/install_lite.py",
        "tests/test_install_lite.py",
        "docs/deployment/lite-install.example.yaml",
    ):
        _write(str(repo / rel), "# abandoned WIP\n")

    proc = _run_hook(repo, tmp_path, hygiene="1")

    assert proc.returncode == 0, f"{proc.stdout}\n{proc.stderr}"
    assert _wip_dirt(repo) == ""
    for rel in (
        "scripts/install_lite.py",
        "tests/test_install_lite.py",
        "docs/deployment/lite-install.example.yaml",
    ):
        assert not (repo / rel).exists(), f"{rel} must be cleaned, not leaked into deploy"


# ===========================================================================
# TC-03 — preservation of rollback/log/gitignored/sibling artefacts (NFR-2)
# ===========================================================================
def test_tc03_preserves_rollback_and_sibling_artifacts(tmp_path):
    repo = _seed_origin_and_clone(tmp_path)
    _write(str(repo / "src" / "config.py"), _DIRTY)  # force the hygiene path

    # In-$REPO artefacts that MUST survive (untracked, NOT gitignored).
    _write(str(repo / ".deploy-prev-image-staging"), "sha256:stagingprev\n")
    _write(str(repo / "deploy-hook.log"), "audit line\n")
    # gitignored prod secrets / DB — must survive `git clean -fd` (NO -x).
    _write(str(repo / ".env"), "ORCH_SECRET=keepme\n")
    _write(str(repo / "data" / "orchestrator.db"), "sqlite-bytes\n")
    # .git internal worktree admin record — git clean never touches .git/.
    _write(str(repo / ".git" / "worktrees" / "wt1" / "HEAD"), "ref: refs/heads/x\n")
    # Sibling state under the PARENT of $REPO — outside the clean scope.
    _write(str(tmp_path / ".deploy-state-orchestrator" / "ORCH-112" / "result"), "0\n")
    _write(str(tmp_path / ".merge-lease-orchestrator.json"), '{"branch":"x"}\n')

    proc = _run_hook(repo, tmp_path, hygiene="1")
    assert proc.returncode == 0, f"{proc.stdout}\n{proc.stderr}"

    # Rollback snapshot freshly written by step 1 (PREV_IMAGE_FILE) survived hygiene.
    assert (repo / ".deploy-prev-image-prod").is_file()
    assert (repo / ".deploy-prev-image-prod").read_text().strip() != ""
    # Wildcard-excluded sibling prev-image + log survived.
    assert (repo / ".deploy-prev-image-staging").read_text() == "sha256:stagingprev\n"
    assert (repo / "deploy-hook.log").read_text() == "audit line\n"
    # gitignored secrets/DB survived (proves NO -x at runtime).
    assert (repo / ".env").read_text() == "ORCH_SECRET=keepme\n"
    assert (repo / "data" / "orchestrator.db").read_text() == "sqlite-bytes\n"
    # .git internal + sibling state untouched.
    assert (repo / ".git" / "worktrees" / "wt1" / "HEAD").is_file()
    assert (tmp_path / ".deploy-state-orchestrator" / "ORCH-112" / "result").is_file()
    assert (tmp_path / ".merge-lease-orchestrator.json").is_file()


# ===========================================================================
# TC-04 — happy-path: genuinely clean base -> hygiene no-op + plain fast-forward.
# Uses prev_running=False so step 1 records NO prev-image, leaving the base clean at
# the hygiene step (no untracked artefact) — the no-op `else` branch is exercised and
# the deploy reduces to the plain `git pull` fast-forward (exit-codes byte-for-byte).
# ===========================================================================
def test_tc04_clean_base_fast_forwards_no_op_hygiene(tmp_path):
    repo = _seed_origin_and_clone(tmp_path)  # repo is CLEAN, just behind origin

    proc = _run_hook(repo, tmp_path, hygiene="1", prev_running=False)

    assert proc.returncode == 0, f"{proc.stdout}\n{proc.stderr}"
    log = (tmp_path / "state" / "hook.log").read_text(encoding="utf-8")
    assert "deploy-base already clean (no-op)" in log
    assert "dirty deploy-base detected" not in log
    # Plain fast-forward brought origin/main's V2.
    assert _head_config(repo) == _V2


# ===========================================================================
# TC-07 — convergence after cancel/failed: leftovers cleared, next deploy clean
# ===========================================================================
def test_tc07_convergence_then_next_deploy_is_clean(tmp_path):
    repo = _seed_origin_and_clone(tmp_path)
    # Leftovers from a failed/cancelled task: dirty tracked + untracked WIP.
    _write(str(repo / "src" / "config.py"), _DIRTY)
    _write(str(repo / "tests" / "test_install_lite.py"), "# wip\n")

    first = _run_hook(repo, tmp_path, hygiene="1")
    assert first.returncode == 0, f"{first.stdout}\n{first.stderr}"
    assert _wip_dirt(repo) == ""  # base converged, no WIP residue
    assert _head_config(repo) == _V2

    # A subsequent self-deploy proceeds without manual intervention (no WIP to block it).
    second = _run_hook(repo, tmp_path, hygiene="1")
    assert second.returncode == 0, f"{second.stdout}\n{second.stderr}"
    assert _wip_dirt(repo) == ""


# ===========================================================================
# TC-05 — self-hosting safety + static hook safety contract (never -x / excludes)
# ===========================================================================
def _hook_code_lines():
    """Non-comment, non-blank lines of the hook (so a comment mentioning `-x` or
    `exit` for documentation does not trip the static safety asserts)."""
    out = []
    for ln in open(HOOK, encoding="utf-8").read().splitlines():
        s = ln.strip()
        if not s or s.startswith("#"):
            continue
        out.append(ln)
    return out


def test_tc05_hook_clean_is_never_destructive():
    text = open(HOOK, encoding="utf-8").read()
    code = "\n".join(_hook_code_lines())
    assert "CHECKOUT_HYGIENE" in text, "hygiene block must exist in the hook"
    # INV-HYGIENE-1: the hook's only `git clean` is `-fd`, NEVER `-x` (which would
    # delete gitignored .env / data/*.db / build/). Checked against CODE only.
    assert "git clean -fd" in code
    assert "-x" not in code  # no -x / -xfd / -fdx in any executable line
    # INV-HYGIENE-2: explicit excludes for the untracked-but-not-ignored artefacts.
    assert "-e '.deploy-prev-image-*'" in code
    assert "-e 'deploy-hook.log'" in code
    # Converge to the authoritative remote, never a local guess.
    assert "git reset --hard origin/main" in code
    # Self-hosting safety: the hygiene path never pushes/force-pushes the remote.
    assert "push --force" not in code and "push -f " not in code


def test_tc05_leaf_is_a_pure_leaf():
    """checkout_hygiene must not import stage_engine / launcher at module load."""
    src = open(os.path.join(ROOT, "src", "checkout_hygiene.py"), encoding="utf-8").read()
    import_lines = [
        ln for ln in src.splitlines()
        if ln.startswith("import ") or ln.startswith("from ")
    ]
    joined = "\n".join(import_lines)
    assert "stage_engine" not in joined
    assert "launcher" not in joined


# ===========================================================================
# TC-06 — kill-switch + repo scope (applies / hook_env)
# ===========================================================================
def test_tc06_kill_switch_off_is_inert(monkeypatch):
    from src.config import settings
    monkeypatch.setattr(settings, "checkout_hygiene_enabled", False)
    assert checkout_hygiene.applies("orchestrator") is False
    assert checkout_hygiene.hook_env("orchestrator", "ORCH-112") == ""


def test_tc06_empty_csv_is_self_hosting_only(monkeypatch):
    from src.config import settings
    monkeypatch.setattr(settings, "checkout_hygiene_enabled", True)
    monkeypatch.setattr(settings, "checkout_hygiene_repos", "")
    assert checkout_hygiene.applies("orchestrator") is True
    assert checkout_hygiene.applies("enduro-trails") is False
    env = checkout_hygiene.hook_env("orchestrator", "ORCH-112")
    assert env.startswith("CHECKOUT_HYGIENE=1 ")
    assert "HYGIENE_REPORT=" in env
    # A non-self repo gets no hygiene env (other repos unaffected).
    assert checkout_hygiene.hook_env("enduro-trails", "ET-1") == ""


def test_tc06_csv_scope_limits_repos(monkeypatch):
    from src.config import settings
    monkeypatch.setattr(settings, "checkout_hygiene_enabled", True)
    monkeypatch.setattr(settings, "checkout_hygiene_repos", "alpha, beta")
    assert checkout_hygiene.applies("alpha") is True
    assert checkout_hygiene.applies("beta") is True
    assert checkout_hygiene.applies("orchestrator") is False


# ===========================================================================
# TC-08 — observability: read_report / alert_dirty never-raise
# ===========================================================================
def test_tc08_read_report_none_when_absent(monkeypatch, tmp_path):
    from src.config import settings
    monkeypatch.setattr(settings, "repos_dir", str(tmp_path))
    assert checkout_hygiene.read_report("orchestrator", "ORCH-112") is None


def test_tc08_read_report_parses_dirty_sentinel(monkeypatch, tmp_path):
    from src import self_deploy
    from src.config import settings
    monkeypatch.setattr(settings, "repos_dir", str(tmp_path))
    d = self_deploy.container_state_dir("orchestrator", "ORCH-112")
    os.makedirs(d, exist_ok=True)
    _write(os.path.join(d, "hygiene"), "dirty=1\n M src/config.py\n?? scripts/x.py\n")
    rep = checkout_hygiene.read_report("orchestrator", "ORCH-112")
    assert rep == {"dirty": True, "paths": ["M src/config.py", "?? scripts/x.py"]}


def test_tc08_alert_dirty_never_raises_on_send_failure(monkeypatch):
    import src.notifications as notifications

    def boom(*a, **k):
        raise RuntimeError("telegram down")

    monkeypatch.setattr(notifications, "send_telegram", boom)
    # Must swallow the error (best-effort) and NOT crash the finalizer.
    assert checkout_hygiene.alert_dirty(
        "orchestrator", "ORCH-112", {"dirty": True, "paths": ["x"]}
    ) is False
    # No report / not dirty -> no alert, no raise.
    assert checkout_hygiene.alert_dirty("orchestrator", "ORCH-112", None) is False


# ===========================================================================
# TC-09 — pipeline invariant: STAGE_TRANSITIONS / QG_CHECKS / exit-codes untouched
# ===========================================================================
def test_tc09_pipeline_contracts_untouched():
    from src import stages
    from src.qg import checks
    # The hygiene feature is NOT a stage and NOT a QG check.
    assert "checkout_hygiene" not in {
        k for tr in stages.STAGE_TRANSITIONS.values() for k in (tr if isinstance(tr, dict) else {})
    }
    assert not any("hygiene" in name for name in checks.QG_CHECKS)


def test_tc09_hook_exit_code_contract_intact():
    text = open(HOOK, encoding="utf-8").read()
    # The hook still maps to the 0/1/2 contract (ORCH-036).
    assert "exit 0" in text and "exit 1" in text and "exit 2" in text
    # The hygiene block itself never emits an `exit` statement (best-effort,
    # never-break). Inspect only the CODE lines of the 2a block (a comment that
    # mentions "exit-codes" must not trip this).
    block = text.split("# 2a.", 1)[1].split("# 2.", 1)[0]
    code_lines = [
        ln for ln in block.splitlines()
        if ln.strip() and not ln.strip().startswith("#")
    ]
    for ln in code_lines:
        assert not ln.strip().startswith("exit"), (
            f"hygiene block must never change the hook exit-code: {ln!r}"
        )


# ===========================================================================
# TC-10 — documentation invariant (golden source)
# ===========================================================================
def test_tc10_docs_state_deploy_base_invariant():
    infra = open(os.path.join(ROOT, "docs", "operations", "INFRA.md"), encoding="utf-8").read()
    readme = open(os.path.join(ROOT, "docs", "architecture", "README.md"), encoding="utf-8").read()
    for doc in (infra, readme):
        assert "ORCH-112" in doc
        assert "deploy/worktree-management база" in doc
        assert "workspace" in doc