admin/orchestrator
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fix(reconciler): terminal-skip + state_uuid dedup on F-1 path

Browse Source 
Закрывает F-1-пробел ORCH-068: терминал-исключение и in-memory dedup
(изначально только F-2) распространены на gate-side путь реконсилятора,
устраняя ложное «🔧 reconciler: ET-002 done разблокирована (потерян
webhook)» (особенно после рестарта).

- D1: новый _resolve_issue_status — один сетевой резолв Plane-статуса
  задачи за тик (states, groups, state_uuid) после дешёвых локальных
  гардов; never-raise -> ({}, {}, None) при сбое.
- D2: безусловный терминал-скип ДО Guard 2 (группа Plane completed/
  cancelled, fallback на логические ключи done/cancelled, либо стадия в
  БД орка ∈ {done, cancelled}); skipped_terminal_total++, не подчинён
  reconcile_skip_blocked_enabled.
- D3: _is_blocked_or_needs_input переиспользует резолв D1 (опц. аргументы,
  _UNSET -> самостоятельный резолв для прямых/легаси-вызовов; 1:1).
- D4: вызов _note_unblock на F-1 теперь передаёт state_uuid -> dedup
  работает на обоих путях (deduped_total++ на повторе).

Анти-регресс: легитимный unblock не-терминальной застрявшей задачи
по-прежнему advance + один Telegram. STAGE_TRANSITIONS / QG_CHECKS /
схема БД / сигнатуры advance_*/_note_unblock / форма status() / новые
флаги — без изменений; never-raise сохранён.

Тесты: tests/test_reconciler.py TC-86-01..09/11,
tests/test_reconciler_plane.py TC-86-10. Полный прогон зелёный (1069).

Refs: ORCH-086
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
claude-bot
2026-06-09 02:17:59 +03:00
committed by stream
parent aa488edddf
commit 595c382ac7
5 changed files with 336 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
src/reconciler.py
View File

@@ -73,6 +73,11 @@ from . import task_deps
logger = logging.getLogger("orchestrator.reconciler")
# ORCH-086 (D3): sentinel distinguishing "caller did not pass a pre-resolved
# state_uuid" (Guard 2 self-resolves, backward-compatible 1-arg call) from an
# explicit ``None`` (Plane unreachable -> conservative skip).
_UNSET = object()
def _parse_grace_overrides(raw: str) -> dict[str, int]:
"""Parse ``reconcile_grace_overrides_json`` into {stage: seconds}.
@@ -183,6 +188,14 @@ class Reconciler:
# AC-16: analysis is a human gate -> owned by F-2, never F-1.
if stage == "analysis":
return
# ORCH-086 D2 (DB-side terminal drift): ``get_active_tasks_for_reconcile``
# filters ``stage != 'done'`` but NOT ``cancelled``. A task already
# terminal in the orchestrator DB is fully in sync by definition -> skip
# before any gate/network work, mirroring the F-2 terminal-skip counter
# (single semantics with ``_reconcile_plane_issue``). Local, no network.
if stage in ("done", "cancelled"):
self.skipped_terminal_total += 1
return
# created / done have no gate to evaluate.
if get_qg_for_stage(stage) is None:
return
@@ -201,9 +214,25 @@ class Reconciler:
# Deterministic, local SQL, no network — and checked FIRST (cheapest).
if developer_retry_count(task_id) >= MAX_DEVELOPER_RETRIES:
return
# ORCH-086 D1: single networked resolve per task per tick, AFTER the cheap
# local guards (so busy/young/escalated tasks never hit Plane). Feeds the
# Plane-side terminal-skip (D2), Guard 2 (D3) and the state_uuid handed to
# _note_unblock (D4) — no duplicate fetch.
states, groups, state_uuid = self._resolve_issue_status(task)
# ORCH-086 D2 (Plane-side terminal-skip), UNCONDITIONAL (not gated by
# reconcile_skip_blocked_enabled, which gates ONLY Guard 2). A task whose
# Plane status is terminal (group completed/cancelled, or the logical
# done/cancelled fallback) is fully in sync -> never a real unblock.
# Runs BEFORE Guard 2 so terminal tasks correctly bump skipped_terminal_total
# instead of being swallowed by Guard 2's conservative path. Closes the F-1
# gap of ORCH-068 (which only covered F-2); fixes the spurious
# "ET-002 ... разблокирована" notification.
if self._is_terminal_state(state_uuid, states, groups):
self.skipped_terminal_total += 1
return
# ORCH-060 Guard 2: respect an explicit human gate (Blocked / Needs Input).
# Networked; runs after Guard 1 so escalated tasks never hit Plane.
if self._is_blocked_or_needs_input(task):
# Reuses the D1 resolve (ORCH-086 D3) so the tick makes a single fetch.
if self._is_blocked_or_needs_input(task, states, state_uuid):
return
# ORCH-026 Guard 3 (B-5): a task blocked by an unfinished declared
# dependency is legitimately waiting, NOT stuck -> F-1 must not advance it
@@ -225,9 +254,48 @@ class Reconciler:
task.get("branch") or "",
)
if result is not None and getattr(result, "advanced", False):
self._note_unblock(task.get("work_item_id") or str(task_id), stage)
# ORCH-086 D4: pass state_uuid so the in-memory dedup guard covers F-1
# too (a repeat tick for the same issue+state is suppressed; survives
# the "first pass after restart" symptom together with the D2 skip).
self._note_unblock(
task.get("work_item_id") or str(task_id), stage, state_uuid
)
def _is_blocked_or_needs_input(self, task: dict) -> bool:
def _resolve_issue_status(
self, task: dict
) -> tuple[dict, dict, str | None]:
"""ORCH-086 D1: one networked resolve per task per tick.
Returns ``(states, groups, current_state_uuid)``. A single
``fetch_issue_state`` plus the cached (ORCH-068 TTL)
``get_project_states`` / ``get_project_state_groups``. The result feeds
the terminal-skip (D2), Guard 2 (D3) and the ``state_uuid`` handed to
``_note_unblock`` (D4), so the tick never fetches the same issue twice.
**never-raise.** On any failure / unresolved project / missing state ->
``({} or states, {} or groups, None)`` so callers apply their
conservative fallback (terminal-skip = not terminal; Guard 2 = skip).
"""
try:
proj = projects.get_project_by_repo(task.get("repo") or "")
if proj is None:
return {}, {}, None
pid = proj.plane_project_id
states = get_project_states(pid)
groups = get_project_state_groups(pid)
issue_id = task.get("plane_id") or task.get("plane_issue_id") or ""
state_uuid = fetch_issue_state(issue_id, pid)
return states or {}, groups or {}, state_uuid
except Exception as e: # noqa: BLE001 - never break the tick
logger.warning(
f"reconciler D1: status resolve failed for task "
f"{task.get('id')}, treating as unresolved: {e}"
)
return {}, {}, None
def _is_blocked_or_needs_input(
self, task: dict, states: dict | None = None, state_uuid=_UNSET
) -> bool:
"""Guard 2 (ORCH-060 + ORCH-066): is this issue waiting for a human OR in
an active orchestrator wait that F-1 must not "revive"?
@@ -251,19 +319,22 @@ class Reconciler:
human-gated task re-introduces the bounce we are trying to kill. The
sub-flag ``reconcile_skip_blocked_enabled`` disables ONLY this networked
guard (escape hatch for a Plane outage); Guard 1 stays active.
**ORCH-086 D3:** the production caller (``_reconcile_gate_task``) passes
the already-resolved ``(states, state_uuid)`` from the single D1 fetch, so
the tick does not hit Plane twice. When ``state_uuid`` is left ``_UNSET``
(direct/legacy 1-arg call) Guard 2 self-resolves via ``_resolve_issue_status``
— behaviour identical to the pre-ORCH-086 code.
"""
if not settings.reconcile_skip_blocked_enabled:
return False
try:
proj = projects.get_project_by_repo(task.get("repo") or "")
if proj is None:
return True # cannot resolve the project -> conservative skip
pid = proj.plane_project_id
states = get_project_states(pid)
issue_id = task.get("plane_id") or task.get("plane_issue_id") or ""
cur = fetch_issue_state(issue_id, pid)
if cur is None:
return True # Plane unreachable / no state -> conservative skip
if state_uuid is _UNSET:
# Backward-compatible self-resolve (direct callers / tests).
states, _groups, state_uuid = self._resolve_issue_status(task)
if not states or state_uuid is None:
return True # unresolved project / Plane unreachable -> conservative skip
cur = state_uuid
# ORCH-066 BR-13: active orchestrator waits, minus base working
# statuses so aliased (enduro) keys never widen the skip-set.
base_working = {