feat(merge-verify): guarantee idempotent open code-PR before merge_pr (ORCH-082)
Close the missing invariant "by merge-verify time the branch has an open code-PR". The pipeline created a PR only on the developer path with a fresh worktree commit (launcher._ensure_pr), so a branch (e.g. after a manual main restore) could reach the deploy->done merge-verify under-gate PR-less -> merge_pr returned "no open PR" -> a FALSE HOLD (ORCH-074 incident). - merge_gate.ensure_open_pr(repo, branch) -> (status, detail): idempotent leaf-actor (never-raise). GET open PRs filtered head==branch AND base==main (identical to merge_pr/ORCH-073 FR-3 — auto docs-PR is not a code-PR) -> existed; else POST -> created; 409/422 race -> re-GET -> existed (no dup); any other error -> failed. - stage_engine._handle_merge_verify: врезка after validated_revision and BEFORE merge_pr. created|existed -> proceed; failed -> honest HOLD via new _hold_pr_create_failed (note "pr-create-failed-hold", text distinguishable from the not-merged HOLD; task stays on deploy, NO rollback). - launcher._ensure_pr delegated to ensure_open_pr (single PR-creation path, shared head==branch & base==main filter); the developer-only trigger is unchanged. - ORCH-073 protection untouched & authoritative: merge is confirmed ONLY by verify_merged_to_main (SHA-in-main) + check_main_regression. Real un-merged code still HOLDs. - Kill-switch ORCH_MERGE_VERIFY_AUTOCREATE_PR_ENABLED (default true); scope = merge_verify_applies (self-hosting / merge_verify_repos); non-self -> no-op; false -> ORCH-074 behaviour 1:1. No DB migration; main never push/force-push. - Append ORCH-082 marker to MAIN_REGRESSION_MARKERS (append-only convention). - conftest defaults the autocreate flag OFF (mirrors merge_verify_enabled) so unrelated deploy->done tests stay 1:1 (no network). Tests: tests/test_orch082_ensure_pr.py (TC-01..05), tests/test_orch082_merge_verify_autocreate.py (TC-06..12). Docs: README merge-verify block (ORCH-082), CHANGELOG, .env.example. Refs: ORCH-082 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
stream
@@ -1077,35 +1077,28 @@ class AgentLauncher:
|
||||
return None
|
||||
|
||||
def _ensure_pr(self, repo: str, branch: str, run_id: int):
|
||||
import httpx
|
||||
owner = settings.gitea_owner
|
||||
headers = {"Authorization": f"token {settings.gitea_token}"}
|
||||
base_url = f"{settings.gitea_url}/api/v1"
|
||||
try:
|
||||
resp = httpx.get(
|
||||
f"{base_url}/repos/{owner}/{repo}/pulls",
|
||||
params={"state": "open", "head": branch},
|
||||
headers=headers, timeout=10
|
||||
)
|
||||
resp.raise_for_status()
|
||||
prs = resp.json()
|
||||
if prs:
|
||||
return prs[0]["number"]
|
||||
parts = branch.split("/")
|
||||
title = parts[-1] if parts else branch
|
||||
resp = httpx.post(
|
||||
f"{base_url}/repos/{owner}/{repo}/pulls",
|
||||
json={"title": f"feat: {title}", "head": branch, "base": "main",
|
||||
"body": f"Auto-created by orchestrator after developer run_id={run_id}"},
|
||||
headers=headers, timeout=10
|
||||
)
|
||||
resp.raise_for_status()
|
||||
pr_number = resp.json()["number"]
|
||||
logger.info(f"Created PR #{pr_number} for {branch}")
|
||||
return pr_number
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create PR for {branch}: {e}")
|
||||
return None
|
||||
"""Ensure an open code-PR exists for ``branch``; return its number or None.
|
||||
|
||||
ORCH-082 (ADR-001 Р-4): delegated to the single idempotent PR-creation actor
|
||||
``merge_gate.ensure_open_pr`` so PR creation lives in ONE place and logs the
|
||||
same created/existed/failed outcomes (G3). The CALL TRIGGER is unchanged — the
|
||||
caller (`_monitor_agent`) still invokes this ONLY on the developer path with a
|
||||
fresh worktree commit; only the implementation under the hood is shared. The
|
||||
actor uses the same ``head==branch AND base==main`` filter as ``merge_pr``, so
|
||||
the developer-created PR and the one merge-verify merges are guaranteed to be
|
||||
the same code-PR. Never raises (the actor is never-raise); ``failed`` -> None,
|
||||
preserving the previous "best-effort, return None on failure" contract.
|
||||
"""
|
||||
from .. import merge_gate
|
||||
status, detail = merge_gate.ensure_open_pr(repo, branch)
|
||||
logger.info(f"_ensure_pr({branch}, run_id={run_id}) -> {status} ({detail})")
|
||||
if status in ("created", "existed"):
|
||||
try:
|
||||
return int(detail)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
logger.error(f"Failed to ensure PR for {branch}: {detail}")
|
||||
return None
|
||||
|
||||
def _write_task_file(self, repo: str, branch: str, task_file: str, content: str):
|
||||
"""Write task file directly into the task's worktree.
|
||||
|
||||
@@ -442,6 +442,22 @@ class Settings(BaseSettings):
|
||||
# merge_verify_repos), so non-self repos are a no-op.
|
||||
regression_guard_enabled: bool = True
|
||||
|
||||
# ORCH-082 (ADR-001 Р-5): guarantee an open code-PR BEFORE the deterministic
|
||||
# merge_pr inside the merge-verify under-gate. The pipeline never guaranteed the
|
||||
# branch had an open PR (head==branch, base==main) at merge time — PRs are created
|
||||
# ONLY on the developer path with a fresh worktree commit (launcher._ensure_pr),
|
||||
# so a branch (e.g. after a manual main restore / a bounce with no new commits)
|
||||
# could reach merge-verify PR-less -> merge_pr returns "no open PR" -> a FALSE HOLD
|
||||
# that ORCH-073 fail-closed correctly catches but should never have to. The
|
||||
# idempotent leaf-actor merge_gate.ensure_open_pr creates/finds the code-PR ДО
|
||||
# merge_pr; ORCH-073's SHA-in-main proof is untouched and stays authoritative.
|
||||
# merge_verify_autocreate_pr_enabled -> kill-switch (env
|
||||
# ORCH_MERGE_VERIFY_AUTOCREATE_PR_ENABLED). False -> exactly the pre-ORCH-082
|
||||
# behaviour (no auto-create; "no open PR" -> HOLD as before). Reuses the
|
||||
# merge_verify_applies scope (self-hosting / merge_verify_repos) — no separate
|
||||
# *_repos, since auto-create is semantically inseparable from merge-verify.
|
||||
merge_verify_autocreate_pr_enabled: bool = True
|
||||
|
||||
# Telegram notifications
|
||||
telegram_bot_token: str = ""
|
||||
telegram_chat_id: str = ""
|
||||
|
||||
@@ -587,6 +587,101 @@ def merge_verify_applies(repo: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def ensure_open_pr(repo: str, branch: str) -> tuple[str, str]:
|
||||
"""Guarantee an open **code-PR** (``head==branch`` AND ``base=="main"``) exists.
|
||||
|
||||
ORCH-082 (ADR-001 Р-1 / FR-1): the idempotent leaf-actor that closes the missing
|
||||
invariant "by merge-verify time the branch has an open code-PR". The pipeline used
|
||||
to create a PR ONLY on the developer path with a fresh worktree commit
|
||||
(``launcher._ensure_pr``), so a branch could reach the ``deploy -> done`` merge-verify
|
||||
under-gate with no open code-PR -> ``merge_pr`` returned ``"no open PR"`` -> a FALSE
|
||||
HOLD (the ORCH-074 incident). This actor creates/finds the code-PR ДО the
|
||||
deterministic ``merge_pr``; ORCH-073's SHA-in-main proof stays authoritative.
|
||||
|
||||
Algorithm (FR-1):
|
||||
1. ``GET …/pulls?state=open`` -> a PR with **``head.ref==branch`` AND
|
||||
``base.ref=="main"``**. The filter is **identical** to ``merge_pr``/ORCH-073
|
||||
FR-3 so both actors agree on exactly the same PR — an auto docs-PR
|
||||
(``base != main``) is NOT a code-PR (AC-6). Found -> ``("existed", "<number>")``.
|
||||
2. Otherwise ``POST …/pulls`` (``head=branch``, ``base=main``, auto title/body) ->
|
||||
``201`` -> ``("created", "<number>")``.
|
||||
3. Idempotency on a race: a ``POST`` that fails because the PR already exists
|
||||
(Gitea ``409``/``422``) -> a repeat ``GET`` (step 1) confirms the existing PR ->
|
||||
``("existed", …)``; no duplicate is created (AC-2 / FR-5).
|
||||
4. Any other HTTP/parse/network error -> ``("failed", "<reason>")``.
|
||||
|
||||
Reuses ``settings.merge_pr_timeout_s`` (same class of Gitea calls as ``merge_pr``).
|
||||
Never-raise (AC-7): any unexpected error -> ``("failed", str(e))``; the exception is
|
||||
NEVER propagated into ``_handle_merge_verify`` / ``advance_stage``.
|
||||
"""
|
||||
try:
|
||||
import httpx
|
||||
owner = settings.gitea_owner
|
||||
headers = {"Authorization": f"token {settings.gitea_token}"}
|
||||
base = f"{settings.gitea_url}/api/v1/repos/{owner}/{repo}"
|
||||
timeout = settings.merge_pr_timeout_s
|
||||
|
||||
def _find_open_code_pr() -> int | None:
|
||||
"""GET open PRs; return the code-PR number (head==branch AND base==main)."""
|
||||
resp = httpx.get(
|
||||
f"{base}/pulls", params={"state": "open"}, headers=headers, timeout=timeout
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
for pr in resp.json() or []:
|
||||
if (
|
||||
pr.get("head", {}).get("ref") == branch
|
||||
and pr.get("base", {}).get("ref") == "main"
|
||||
):
|
||||
return pr.get("number")
|
||||
return None
|
||||
|
||||
# Step 1: an open code-PR already exists -> existed (no duplicate POST).
|
||||
existing = _find_open_code_pr()
|
||||
if existing is not None:
|
||||
logger.info("ensure_open_pr: %s/%s already has open code-PR #%s", repo, branch, existing)
|
||||
return "existed", str(existing)
|
||||
|
||||
# Step 2: create the code-PR onto main.
|
||||
parts = branch.split("/")
|
||||
title = parts[-1] if parts else branch
|
||||
m = httpx.post(
|
||||
f"{base}/pulls",
|
||||
json={
|
||||
"title": f"feat: {title}",
|
||||
"head": branch,
|
||||
"base": "main",
|
||||
"body": f"Auto-created by orchestrator merge-verify for {branch}",
|
||||
},
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
)
|
||||
if m.status_code in (200, 201):
|
||||
number = (m.json() or {}).get("number")
|
||||
logger.info("ensure_open_pr: created PR #%s for %s/%s", number, repo, branch)
|
||||
return "created", str(number)
|
||||
|
||||
# Step 3: race / already-exists (409 conflict, 422 unprocessable) -> re-GET.
|
||||
if m.status_code in (409, 422):
|
||||
again = _find_open_code_pr()
|
||||
if again is not None:
|
||||
logger.info(
|
||||
"ensure_open_pr: %s/%s PR already existed on retry (#%s, HTTP %s)",
|
||||
repo, branch, again, m.status_code,
|
||||
)
|
||||
return "existed", str(again)
|
||||
|
||||
detail = (m.text or "").strip()[:200]
|
||||
logger.warning(
|
||||
"ensure_open_pr: create failed for %s/%s: HTTP %s %s",
|
||||
repo, branch, m.status_code, detail,
|
||||
)
|
||||
return "failed", f"create PR failed: HTTP {m.status_code}"
|
||||
except Exception as e: # noqa: BLE001 - never-raise contract (AC-7)
|
||||
logger.warning("ensure_open_pr unexpected error for %s/%s: %s", repo, branch, e)
|
||||
return "failed", f"ensure_open_pr error: {e}"
|
||||
|
||||
|
||||
def merge_pr(repo: str, branch: str) -> tuple[bool, str]:
|
||||
"""Deterministically merge the open PR for ``branch`` via the Gitea PR-merge API.
|
||||
|
||||
@@ -730,6 +825,7 @@ MAIN_REGRESSION_MARKERS: list[tuple[str, str, str]] = [
|
||||
("ORCH-069", "qg0_title_max", "src/config.py"),
|
||||
("ORCH-071", "verify_merged_to_main", "src/merge_gate.py"),
|
||||
("ORCH-073", "check_main_regression", "src/merge_gate.py"),
|
||||
("ORCH-082", "ensure_open_pr", "src/merge_gate.py"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -1321,6 +1321,52 @@ def _hold_main_regressed(
|
||||
return True
|
||||
|
||||
|
||||
def _hold_pr_create_failed(
|
||||
task_id, repo, work_item_id, branch, reason: str, result: AdvanceResult
|
||||
) -> bool:
|
||||
"""HOLD the task because the open code-PR could not be ensured (ORCH-082 Р-3).
|
||||
|
||||
FR-2/FR-4 (AC-5/AC-7): ``ensure_open_pr`` returned ``"failed"`` (Gitea unreachable /
|
||||
HTTP error) — there is no open code-PR and one could not be created. Symmetric to the
|
||||
not-merged / regressed HOLD: task stays on ``deploy`` (NOT done), NO rollback to
|
||||
development, ALERT-only (Telegram + Plane ``set_issue_blocked`` + comment). The HOLD
|
||||
text MUST be distinguishable from the not-merged HOLD so the operator sees the cause is
|
||||
"could not CREATE the PR" (infra), not "could not MERGE an existing one". Returns
|
||||
``True`` (INTERVENED). Never breaks the HOLD on a notify error; ``failed`` is a
|
||||
structured outcome, not a propagated exception (INV-1).
|
||||
"""
|
||||
merge_gate.note_not_merged_alert(work_item_id) # reuse the counter-notifier.
|
||||
msg = (
|
||||
f"PR создать не удалось: {reason} (repo={repo}, branch={branch}, "
|
||||
f"wi={work_item_id}). Открытый код-PR отсутствует и не создан — задача "
|
||||
f"удержана на `deploy` (НЕ done). Нужно проверить доступность Gitea / создать PR."
|
||||
)
|
||||
logger.warning(f"Task {task_id}: {msg}")
|
||||
if work_item_id:
|
||||
try:
|
||||
set_issue_blocked(work_item_id)
|
||||
except Exception as e: # noqa: BLE001 - never break the HOLD
|
||||
logger.warning(f"Task {task_id}: set_issue_blocked failed: {e}")
|
||||
try:
|
||||
plane_add_comment(
|
||||
work_item_id,
|
||||
"\U0001f6a8 PR создать не удалось: " + reason + ". Открытый код-PR "
|
||||
"отсутствует — задача удержана на `deploy` (НЕ done). Проверьте "
|
||||
"доступность Gitea / создайте PR вручную и повторите approve.",
|
||||
author="deployer",
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - never break the HOLD
|
||||
logger.warning(f"Task {task_id}: plane pr-create-failed comment failed: {e}")
|
||||
try:
|
||||
send_telegram(f"\U0001f6a8 {msg}")
|
||||
except Exception as e: # noqa: BLE001 - never break the HOLD
|
||||
logger.warning(f"Task {task_id}: pr-create-failed telegram failed: {e}")
|
||||
result.alerted = True
|
||||
result.note = "pr-create-failed-hold"
|
||||
result.advanced = False
|
||||
return True
|
||||
|
||||
|
||||
def _handle_merge_verify(task_id, repo, work_item_id, branch, result: AdvanceResult) -> bool:
|
||||
"""ORCH-071 merge-verify under-gate on the `deploy -> done` edge.
|
||||
|
||||
@@ -1353,6 +1399,24 @@ def _handle_merge_verify(task_id, repo, work_item_id, branch, result: AdvanceRes
|
||||
from . import image_freshness
|
||||
sha = image_freshness.validated_revision(repo, branch)
|
||||
|
||||
# ORCH-082 (Р-2 / FR-2): guarantee an open code-PR (head==branch, base==main)
|
||||
# BEFORE the deterministic merge_pr. The pipeline never guaranteed the branch
|
||||
# had one at merge time (PRs are created only on the developer path with a fresh
|
||||
# commit) -> a PR-less branch hit merge_pr "no open PR" -> a FALSE HOLD (ORCH-074).
|
||||
# `created`/`existed` -> proceed unchanged; `failed` -> honest HOLD with a
|
||||
# distinguishable text (NOT the not-merged HOLD). ORCH-073's SHA-in-main proof
|
||||
# below is untouched and stays authoritative. Kill-switch off -> 1:1 prior path.
|
||||
if settings.merge_verify_autocreate_pr_enabled:
|
||||
pr_status, pr_detail = merge_gate.ensure_open_pr(repo, branch)
|
||||
logger.info(
|
||||
f"Task {task_id}: merge-verify ensure_open_pr -> {pr_status} ({pr_detail})"
|
||||
)
|
||||
if pr_status == "failed":
|
||||
return _hold_pr_create_failed(
|
||||
task_id, repo, work_item_id, branch, pr_detail, result
|
||||
)
|
||||
# "created" | "existed" -> proceed normally to merge_pr.
|
||||
|
||||
# Deterministic merge-actor (no-op if the PR is already merged, INV-5/AC-9).
|
||||
merged_ok, merge_msg = merge_gate.merge_pr(repo, branch)
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user