fix(ci): bounce task back to developer on red CI (capped retries) #18
@@ -217,9 +217,30 @@ async def handle_ci_status(payload: dict):
|
||||
notify_qg_failure(task_id, current_stage, "check_ci_green", reason)
|
||||
|
||||
elif state == "failure" and current_stage == "development":
|
||||
# CI is now the authoritative gate for development -> review.
|
||||
# A failing CI means the QG did not pass; notify (do not silently advance).
|
||||
# CI is the authoritative gate for development -> review.
|
||||
# On red CI: notify, then bounce the task back to the developer (capped retries),
|
||||
# symmetric to the review REQUEST_CHANGES path.
|
||||
notify_qg_failure(task_id, current_stage, "check_ci_green", f"Gitea CI failed on branch '{branch}'")
|
||||
conn = get_db()
|
||||
retry_count = conn.execute(
|
||||
"SELECT COUNT(*) as cnt FROM agent_runs WHERE task_id = ? AND agent = 'developer'",
|
||||
(task_id,),
|
||||
).fetchone()["cnt"]
|
||||
conn.close()
|
||||
if retry_count < MAX_DEV_RETRIES:
|
||||
# task already on 'development' — no stage change needed, just relaunch developer
|
||||
try:
|
||||
task_desc = (
|
||||
f"Work item: {work_item_id}\nRepo: {repo_name}\nBranch: {branch}\n"
|
||||
f"Stage: development\nNote: CI failed, fix and re-push (attempt {retry_count + 1}/{MAX_DEV_RETRIES})"
|
||||
)
|
||||
job_id = enqueue_job("developer", repo_name, task_desc, task_id=task_id)
|
||||
logger.info(f"Task {task_id}: CI failed, enqueued developer (attempt {retry_count + 1}, job_id={job_id})")
|
||||
except Exception as e:
|
||||
notify_error(task_id, f"Failed to relaunch developer after CI failure: {e}")
|
||||
else:
|
||||
notify_error(task_id, f"Max developer retries ({MAX_DEV_RETRIES}) reached after CI failure, escalating")
|
||||
logger.error(f"Task {task_id}: max retries reached after CI failure, needs manual intervention")
|
||||
|
||||
|
||||
async def handle_pr(payload: dict):
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import pytest
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
from unittest.mock import patch, MagicMock, AsyncMock
|
||||
@@ -341,3 +342,94 @@ def test_plane_webhook_event_logged():
|
||||
conn.close()
|
||||
assert event is not None
|
||||
assert event["source"] == "plane"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BUG 7: red CI on development must bounce the task back to the developer
|
||||
# (capped retries, symmetric to review REQUEST_CHANGES). These are pure-logic
|
||||
# tests: they invoke handle_ci_status() directly with mocked helpers so they do
|
||||
# not pass through the TestClient HMAC barrier (baseline 401s are off-limits).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _ci_failure_payload():
|
||||
return {
|
||||
"state": "failure",
|
||||
"branches": [{"name": "feature/ET-011-test"}],
|
||||
"repository": {"name": "enduro-trails"},
|
||||
}
|
||||
|
||||
|
||||
def _mock_db_with_retry_count(count):
|
||||
"""Build a get_db() mock whose retry_count query returns `count`."""
|
||||
conn = MagicMock()
|
||||
conn.execute.return_value.fetchone.return_value = {"cnt": count}
|
||||
return conn
|
||||
|
||||
|
||||
@patch("src.webhooks.gitea.notify_error")
|
||||
@patch("src.webhooks.gitea.notify_qg_failure")
|
||||
@patch("src.webhooks.gitea.enqueue_job")
|
||||
@patch("src.webhooks.gitea.update_task_stage")
|
||||
@patch("src.webhooks.gitea.get_db")
|
||||
@patch("src.webhooks.gitea.get_task_by_repo_branch")
|
||||
@patch("src.webhooks.gitea.get_project_by_repo")
|
||||
def test_ci_failure_development_retries_developer_under_limit(
|
||||
mock_proj, mock_task, mock_get_db, mock_update_stage,
|
||||
mock_enqueue, mock_qg, mock_err,
|
||||
):
|
||||
"""retry_count < MAX_DEV_RETRIES → relaunch developer, stage untouched."""
|
||||
from src.webhooks.gitea import handle_ci_status
|
||||
|
||||
mock_proj.return_value = {"repo": "enduro-trails"}
|
||||
mock_task.return_value = {
|
||||
"id": 1, "stage": "development", "work_item_id": "ET-011",
|
||||
}
|
||||
mock_get_db.return_value = _mock_db_with_retry_count(0)
|
||||
mock_enqueue.return_value = 42
|
||||
|
||||
asyncio.run(handle_ci_status(_ci_failure_payload()))
|
||||
|
||||
# QG failure was still reported (Slava sees both the failure and the retry).
|
||||
assert mock_qg.called
|
||||
# developer was re-enqueued.
|
||||
assert mock_enqueue.called
|
||||
assert mock_enqueue.call_args[0][0] == "developer"
|
||||
# No escalation.
|
||||
assert not mock_err.called
|
||||
# Stage stays on development — no update_task_stage in the CI-failure path.
|
||||
assert not mock_update_stage.called
|
||||
|
||||
|
||||
@patch("src.webhooks.gitea.notify_error")
|
||||
@patch("src.webhooks.gitea.notify_qg_failure")
|
||||
@patch("src.webhooks.gitea.enqueue_job")
|
||||
@patch("src.webhooks.gitea.update_task_stage")
|
||||
@patch("src.webhooks.gitea.get_db")
|
||||
@patch("src.webhooks.gitea.get_task_by_repo_branch")
|
||||
@patch("src.webhooks.gitea.get_project_by_repo")
|
||||
def test_ci_failure_development_escalates_at_limit(
|
||||
mock_proj, mock_task, mock_get_db, mock_update_stage,
|
||||
mock_enqueue, mock_qg, mock_err,
|
||||
):
|
||||
"""retry_count >= MAX_DEV_RETRIES → escalate via notify_error, no relaunch."""
|
||||
from src.webhooks.gitea import handle_ci_status, MAX_DEV_RETRIES
|
||||
|
||||
mock_proj.return_value = {"repo": "enduro-trails"}
|
||||
mock_task.return_value = {
|
||||
"id": 1, "stage": "development", "work_item_id": "ET-011",
|
||||
}
|
||||
mock_get_db.return_value = _mock_db_with_retry_count(MAX_DEV_RETRIES)
|
||||
|
||||
asyncio.run(handle_ci_status(_ci_failure_payload()))
|
||||
|
||||
# QG failure still reported.
|
||||
assert mock_qg.called
|
||||
# developer NOT re-enqueued at the cap.
|
||||
assert not mock_enqueue.called
|
||||
# Escalation message mentions CI failure.
|
||||
assert mock_err.called
|
||||
err_msg = " ".join(str(a) for a in mock_err.call_args[0])
|
||||
assert "Max developer retries" in err_msg
|
||||
assert "after CI failure" in err_msg
|
||||
# Stage untouched.
|
||||
assert not mock_update_stage.called
|
||||
|
||||
Reference in New Issue
Block a user