"""ORCH-41: tests for resolve_agent_model (per-agent + per-project LLM model). Covers the 4-level resolution priority: 1. ProjectConfig.agent_models[agent] (per-project override, from projects_json) 2. settings.agent_model_ (per-agent env, when non-empty) 3. settings.agent_model_default (global default) 4. "" (no override anywhere -> CLI default) plus: unknown project_id / no project_id skips level 1, unknown agent skips level 2, and the frozen ProjectConfig still accepts agent_models (default {}). We never mutate the module-global registry permanently: tests that need a custom registry install one via monkeypatch + reload_projects and restore the default afterwards (autouse fixture). """ import os import tempfile import pytest os.environ.setdefault("ORCH_DB_PATH", os.path.join(tempfile.gettempdir(), "test_orch41_model.db")) os.environ.setdefault("ORCH_GITEA_TOKEN", "test-token") os.environ.setdefault("ORCH_PLANE_API_TOKEN", "test-token") from src.agents.launcher import resolve_agent_model from src.config import settings from src import projects as P from src.projects import ProjectConfig, reload_projects, _parse_projects_json ORCH_PLANE_ID = "8da6aa25-a60e-44d6-a1e2-d8ae59aa7d6a" ENDURO_PLANE_ID = "7a79f0a9-5278-49cd-9007-9a338f238f9c" @pytest.fixture(autouse=True) def _clean_settings(monkeypatch): """Reset all per-agent/default model settings to a known baseline so tests are order-independent regardless of what other modules set in the env.""" monkeypatch.setattr(settings, "agent_model_default", "claude-opus-4-8") for a in ("analyst", "architect", "developer", "reviewer", "tester", "deployer"): monkeypatch.setattr(settings, f"agent_model_{a}", "") # default registry (no per-project overrides) monkeypatch.setattr(P.settings, "projects_json", "") reload_projects() yield reload_projects() def _install_registry(monkeypatch, agent_models): """Install a single-project registry for ORCH with the given agent_models.""" reg = [ProjectConfig( plane_project_id=ORCH_PLANE_ID, repo="orchestrator", work_item_prefix="ORCH", name="orchestrator", agent_models=agent_models, )] monkeypatch.setattr(P, "PROJECTS", reg) monkeypatch.setattr(P, "_BY_PLANE_ID", {p.plane_project_id: p for p in reg}) monkeypatch.setattr(P, "_BY_REPO", {p.repo: p for p in reg}) # ---- Level 4: nothing configured -> "" -------------------------------------- def test_no_config_returns_empty(monkeypatch): monkeypatch.setattr(settings, "agent_model_default", "") assert resolve_agent_model("developer") == "" assert resolve_agent_model("developer", ORCH_PLANE_ID) == "" # ---- Level 3: global default ------------------------------------------------ def test_global_default(): assert resolve_agent_model("developer") == "claude-opus-4-8" assert resolve_agent_model("architect") == "claude-opus-4-8" # ---- Level 2: per-agent env beats default ----------------------------------- def test_per_agent_env_overrides_default(monkeypatch): monkeypatch.setattr(settings, "agent_model_reviewer", "claude-sonnet-4-6") assert resolve_agent_model("reviewer") == "claude-sonnet-4-6" # other agents still fall through to default assert resolve_agent_model("developer") == "claude-opus-4-8" # ---- Level 1: per-project override beats per-agent env and default ---------- def test_project_override_beats_env_and_default(monkeypatch): monkeypatch.setattr(settings, "agent_model_developer", "claude-sonnet-4-6") _install_registry(monkeypatch, {"developer": "claude-opus-4-8"}) assert resolve_agent_model("developer", ORCH_PLANE_ID) == "claude-opus-4-8" # without project_id, falls back to per-agent env assert resolve_agent_model("developer") == "claude-sonnet-4-6" def test_project_override_only_for_listed_agent(monkeypatch): _install_registry(monkeypatch, {"developer": "claude-opus-4-8"}) # reviewer not in agent_models -> falls back to default assert resolve_agent_model("reviewer", ORCH_PLANE_ID) == "claude-opus-4-8" monkeypatch.setattr(settings, "agent_model_reviewer", "claude-sonnet-4-6") assert resolve_agent_model("reviewer", ORCH_PLANE_ID) == "claude-sonnet-4-6" # ---- unknown / empty project id skips level 1 ------------------------------- def test_unknown_project_id_skips_override(monkeypatch): _install_registry(monkeypatch, {"developer": "x-model"}) assert resolve_agent_model("developer", "no-such-uuid") == "claude-opus-4-8" assert resolve_agent_model("developer", None) == "claude-opus-4-8" # ---- unknown agent skips per-agent env, still gets default ------------------ def test_unknown_agent_falls_to_default(): assert resolve_agent_model("nonexistent") == "claude-opus-4-8" # ---- frozen ProjectConfig accepts agent_models ------------------------------ def test_projectconfig_frozen_with_agent_models(): pc = ProjectConfig( plane_project_id="x", repo="r", work_item_prefix="P", name="n", agent_models={"developer": "m"}, ) assert pc.agent_models == {"developer": "m"} # default is an empty dict, not shared/mutable across instances pc2 = ProjectConfig(plane_project_id="y", repo="r2", work_item_prefix="P2", name="n2") assert pc2.agent_models == {} assert pc2.agent_models is not pc.agent_models with pytest.raises(Exception): pc.repo = "changed" # frozen # ---- projects_json parsing of agent_models / agent_efforts ------------------ def test_parse_projects_json_with_overrides(): raw = ( '[{"plane_project_id":"p1","repo":"orchestrator",' '"work_item_prefix":"ORCH",' '"agent_models":{"developer":"claude-opus-4-8","reviewer":"claude-sonnet-4-6"},' '"agent_efforts":{"developer":"xhigh","tester":"low"}}]' ) parsed = _parse_projects_json(raw) assert parsed is not None and len(parsed) == 1 pc = parsed[0] assert pc.agent_models == {"developer": "claude-opus-4-8", "reviewer": "claude-sonnet-4-6"} assert pc.agent_efforts == {"developer": "xhigh", "tester": "low"} def test_parse_projects_json_omitted_overrides_default_empty(): raw = ('[{"plane_project_id":"p1","repo":"r","work_item_prefix":"P"}]') parsed = _parse_projects_json(raw) assert parsed is not None and len(parsed) == 1 assert parsed[0].agent_models == {} assert parsed[0].agent_efforts == {} def test_parse_projects_json_malformed_override_ignored(): # agent_models is not an object -> dropped to {}, entry still valid raw = ('[{"plane_project_id":"p1","repo":"r","work_item_prefix":"P",' '"agent_models":"oops"}]') parsed = _parse_projects_json(raw) assert parsed is not None and parsed[0].agent_models == {}