Merge pull request #21 from BraveNewCapital/copilot/update-repo-architect-model-selection

Author: Steake

.github/workflows/repo-architect.yml

@@ -21,7 +21,7 @@ on:
       github_fallback_model:
         description: 'GitHub Models fallback model id (used if primary model fails)'
         required: false
-        default: 'openai/gpt-5'
+        default: ''
         type: string
       report_path:
         description: 'Primary report path'
@@ -85,18 +85,20 @@ jobs:
       - name: Resolve GitHub Models configuration
         env:
           GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_MODEL: ${{ github.event.inputs.github_model }}
+          GITHUB_FALLBACK_MODEL: ${{ github.event.inputs.github_fallback_model }}
         run: |
           python - <<'PY'
           import json
           import os
           import urllib.request
 
+          env_github_model = os.environ.get("GITHUB_MODEL", "").strip()
+          env_github_fallback_model = os.environ.get("GITHUB_FALLBACK_MODEL", "").strip()
           order = [
-              "anthropic/claude-sonnet-4.6",
-              "anthropic/claude-sonnet-4.5",
-              "openai/gpt-4.1",
+              "openai/gpt-5",
+              "openai/o3",
           ]
-          secondary = "google/gemini-3-pro"
           available = set()
           catalog_ok = False
           try:
@@ -131,31 +133,28 @@ jobs:
               candidates = sorted(m for m in available if m != exclude)
               return candidates[0] if candidates else None
 
-          if catalog_ok and available:
-              preferred = (
-                  first_available(order)
-                  or (secondary if secondary in available else None)
-                  or deterministic_available()
-              )
+          if env_github_model:
+              preferred = env_github_model
+          elif catalog_ok and available:
+              preferred = first_available(order) or deterministic_available() or order[0]
           else:
               preferred = order[0]
 
-          if catalog_ok and available:
-              if secondary in available and secondary != preferred:
-                  fallback = secondary
-              else:
-                  fallback = (
-                      first_available([c for c in order if c != preferred])
-                      or deterministic_available(exclude=preferred)
-                      or preferred
-                  )
+          if env_github_fallback_model:
+              fallback = env_github_fallback_model
+          elif catalog_ok and available:
+              fallback = (
+                  first_available([c for c in order if c != preferred])
+                  or deterministic_available(exclude=preferred)
+                  or preferred
+              )
           else:
-              fallback = secondary
+              fallback = order[1] if len(order) > 1 and order[1] != preferred else order[0]
 
           if not isinstance(preferred, str) or not preferred:
               preferred = order[0]
           if not isinstance(fallback, str) or not fallback:
-              fallback = secondary if secondary != preferred else order[-1]
+              fallback = order[1] if len(order) > 1 and order[1] != preferred else order[0]
 
           env_file = os.environ.get("GITHUB_ENV")
           if not env_file:

repo_architect.py

@@ -18,7 +18,7 @@
 import urllib.error
 import urllib.parse
 import urllib.request
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
 
 APP_NAME = "repo-architect"
 VERSION = "2.1.0"
@@ -60,8 +60,10 @@
 }
 
 # Model selection defaults
-DEFAULT_PREFERRED_MODEL = "anthropic/claude-sonnet-4.6"
-DEFAULT_FALLBACK_MODEL = "google/gemini-3-pro"
+DEFAULT_PREFERRED_MODEL = "openai/gpt-5"
+DEFAULT_FALLBACK_MODEL = "openai/o3"
+# Preferred resolution order for automatic model selection
+PREFERRED_MODEL_ORDER: Tuple[str, ...] = ("openai/gpt-5", "openai/o3")
 # Substrings in HTTP error bodies that indicate the model itself is unavailable (not a transient error)
 _MODEL_UNAVAILABLE_SIGNALS = frozenset({
     "unknown_model", "model_not_found", "unsupported_model", "unsupported model",
@@ -347,6 +349,63 @@ def model_available(catalog: List[Dict[str, Any]], model: str) -> bool:
     return False
 
 
+def _resolve_models(
+    available: Set[str],
+    catalog_ok: bool,
+    env_model: str = "",
+    env_fallback: str = "",
+    order: Sequence[str] = PREFERRED_MODEL_ORDER,
+) -> Tuple[str, str]:
+    """Resolve preferred and fallback model IDs using override-first, then catalog-order logic.
+
+    Args:
+        available: Set of model IDs returned by the catalog.
+        catalog_ok: Whether the catalog fetch succeeded.
+        env_model: Explicit primary override (GITHUB_MODEL env var).  Empty → auto-resolve.
+        env_fallback: Explicit fallback override (GITHUB_FALLBACK_MODEL env var).  Empty → auto-resolve.
+        order: Preferred resolution order; defaults to PREFERRED_MODEL_ORDER.
+
+    Returns:
+        (preferred, fallback) — never the same value for both unless only one model exists.
+    """
+    order_list = list(order)
+
+    def first_available(candidates: Sequence[str]) -> Optional[str]:
+        for c in candidates:
+            if c in available:
+                return c
+        return None
+
+    def deterministic_available(exclude: Optional[str] = None) -> Optional[str]:
+        candidates = sorted(m for m in available if m != exclude)
+        return candidates[0] if candidates else None
+
+    if env_model:
+        preferred = env_model
+    elif catalog_ok and available:
+        preferred = first_available(order_list) or deterministic_available() or order_list[0]
+    else:
+        preferred = order_list[0]
+
+    if env_fallback:
+        fallback = env_fallback
+    elif catalog_ok and available:
+        fallback = (
+            first_available([c for c in order_list if c != preferred])
+            or deterministic_available(exclude=preferred)
+            or preferred
+        )
+    else:
+        fallback = order_list[1] if len(order_list) > 1 and order_list[1] != preferred else order_list[0]
+
+    if not isinstance(preferred, str) or not preferred:
+        preferred = order_list[0]
+    if not isinstance(fallback, str) or not fallback:
+        fallback = order_list[1] if len(order_list) > 1 and order_list[1] != preferred else order_list[0]
+
+    return preferred, fallback
+
+
 def github_models_chat(token: str, model: str, messages: List[Dict[str, str]]) -> Dict[str, Any]:
     req = urllib.request.Request(
         GITHUB_MODELS_CHAT,
@@ -1517,14 +1576,14 @@ def workflow_yaml(secret_env_names: Sequence[str], cron: str, github_model: Opti
           - mutate
           - campaign
       github_model:
-        description: 'GitHub Models model id (overrides preferred model)'
+        description: 'GitHub Models model id (overrides preferred model; leave blank to use catalog resolution)'
         required: false
-        default: 'anthropic/claude-sonnet-4.5'
+        default: ''
         type: string
       github_fallback_model:
         description: 'GitHub Models fallback model id (used if primary model fails)'
         required: false
-        default: 'openai/gpt-5'
+        default: ''
         type: string
       report_path:
         description: 'Primary report path'
@@ -1588,18 +1647,20 @@ def workflow_yaml(secret_env_names: Sequence[str], cron: str, github_model: Opti
       - name: Resolve GitHub Models configuration
         env:
           GITHUB_TOKEN: ${{{{ github.token }}}}
+          GITHUB_MODEL: ${{{{ github.event.inputs.github_model }}}}
+          GITHUB_FALLBACK_MODEL: ${{{{ github.event.inputs.github_fallback_model }}}}
         run: |
           python - <<'PY'
           import json
           import os
           import urllib.request
 
+          env_github_model = os.environ.get("GITHUB_MODEL", "").strip()
+          env_github_fallback_model = os.environ.get("GITHUB_FALLBACK_MODEL", "").strip()
           order = [
-              "anthropic/claude-sonnet-4.6",
-              "anthropic/claude-sonnet-4.5",
-              "openai/gpt-4.1",
+              "openai/gpt-5",
+              "openai/o3",
           ]
-          secondary = "google/gemini-3-pro"
           available = set()
           catalog_ok = False
           try:
@@ -1634,31 +1695,28 @@ def deterministic_available(exclude=None):
               candidates = sorted(m for m in available if m != exclude)
               return candidates[0] if candidates else None
 
-          if catalog_ok and available:
-              preferred = (
-                  first_available(order)
-                  or (secondary if secondary in available else None)
-                  or deterministic_available()
-              )
+          if env_github_model:
+              preferred = env_github_model
+          elif catalog_ok and available:
+              preferred = first_available(order) or deterministic_available() or order[0]
           else:
               preferred = order[0]
 
-          if catalog_ok and available:
-              if secondary in available and secondary != preferred:
-                  fallback = secondary
-              else:
-                  fallback = (
-                      first_available([c for c in order if c != preferred])
-                      or deterministic_available(exclude=preferred)
-                      or preferred
-                  )
+          if env_github_fallback_model:
+              fallback = env_github_fallback_model
+          elif catalog_ok and available:
+              fallback = (
+                  first_available([c for c in order if c != preferred])
+                  or deterministic_available(exclude=preferred)
+                  or preferred
+              )
           else:
-              fallback = secondary
+              fallback = order[1] if len(order) > 1 and order[1] != preferred else order[0]
 
           if not isinstance(preferred, str) or not preferred:
               preferred = order[0]
           if not isinstance(fallback, str) or not fallback:
-              fallback = secondary if secondary != preferred else order[-1]
+              fallback = order[1] if len(order) > 1 and order[1] != preferred else order[0]
 
           env_file = os.environ.get("GITHUB_ENV")
           if not env_file:

tests/test_repo_architect.py

@@ -427,14 +427,14 @@ class TestModelConfiguration(unittest.TestCase):
     def test_build_config_uses_env_models_when_github_model_blank(self) -> None:
         env = dict(os.environ)
         env.pop("GITHUB_MODEL", None)
-        env["REPO_ARCHITECT_PREFERRED_MODEL"] = "anthropic/claude-sonnet-4.6"
-        env["REPO_ARCHITECT_FALLBACK_MODEL"] = "google/gemini-3-pro"
+        env["REPO_ARCHITECT_PREFERRED_MODEL"] = "openai/gpt-5"
+        env["REPO_ARCHITECT_FALLBACK_MODEL"] = "openai/o3"
         with patch.object(ra, "discover_git_root", return_value=pathlib.Path("/tmp/repo")):
             with patch.dict(os.environ, env, clear=True):
                 config = ra.build_config(ra.parse_args([]))
         self.assertIsNone(config.github_model)
-        self.assertEqual(config.preferred_model, "anthropic/claude-sonnet-4.6")
-        self.assertEqual(config.fallback_model, "google/gemini-3-pro")
+        self.assertEqual(config.preferred_model, "openai/gpt-5")
+        self.assertEqual(config.fallback_model, "openai/o3")
 
     def test_github_model_override_takes_precedence_over_preferred(self) -> None:
         analysis = {
@@ -470,10 +470,12 @@ def test_workflow_yaml_resolves_models_via_catalog_and_keeps_blank_override_logi
         self.assertIn("Resolve GitHub Models configuration", workflow)
         self.assertIn("https://models.github.ai/catalog/models", workflow)
         self.assertIn("catalog_ok = False", workflow)
-        self.assertIn('"anthropic/claude-sonnet-4.6"', workflow)
-        self.assertIn('"anthropic/claude-sonnet-4.5"', workflow)
-        self.assertIn('"openai/gpt-4.1"', workflow)
-        self.assertIn('secondary = "google/gemini-3-pro"', workflow)
+        self.assertIn('"openai/gpt-5"', workflow)
+        self.assertIn('"openai/o3"', workflow)
+        self.assertNotIn('"anthropic/claude-sonnet-4.6"', workflow)
+        self.assertNotIn('"anthropic/claude-sonnet-4.5"', workflow)
+        self.assertNotIn('"openai/gpt-4.1"', workflow)
+        self.assertNotIn('secondary = "google/gemini-3-pro"', workflow)
         self.assertIn("def deterministic_available(exclude=None):", workflow)
         self.assertIn("or deterministic_available()", workflow)
         self.assertIn("or deterministic_available(exclude=preferred)", workflow)
@@ -482,12 +484,15 @@ def test_workflow_yaml_resolves_models_via_catalog_and_keeps_blank_override_logi
         self.assertIn("REPO_ARCHITECT_FALLBACK_MODEL={fallback}", workflow)
         self.assertIn('if [ -n "$MODEL" ]; then EXTRA_ARGS="$EXTRA_ARGS --github-model $MODEL"; fi', workflow)
         self.assertIn("models: read", workflow)
-        # New: primary/fallback model inputs with defaults
+        # github_fallback_model input with empty default
         self.assertIn("github_fallback_model:", workflow)
-        self.assertIn("default: 'openai/gpt-5'", workflow)
-        self.assertIn("default: 'anthropic/claude-sonnet-4.5'", workflow)
-        # New: GITHUB_MODEL and GITHUB_FALLBACK_MODEL exported as env vars
+        self.assertIn("default: ''", workflow)
+        # GITHUB_MODEL and GITHUB_FALLBACK_MODEL exported as env vars to resolver step
         self.assertIn("GITHUB_FALLBACK_MODEL:", workflow)
+        self.assertIn("GITHUB_MODEL:", workflow)
+        # Override-first logic present
+        self.assertIn("env_github_model", workflow)
+        self.assertIn("env_github_fallback_model", workflow)
 
     def test_build_config_reads_github_fallback_model_env(self) -> None:
         """GITHUB_FALLBACK_MODEL env var should populate github_fallback_model in Config."""
@@ -564,6 +569,80 @@ def test_enrich_metadata_primary_success(self) -> None:
         self.assertIsNone(meta["fallback_reason"])
 
 
+# ---------------------------------------------------------------------------
+# 3b. Model resolver order
+# ---------------------------------------------------------------------------
+
+class TestModelResolverOrder(unittest.TestCase):
+    """Tests for the _resolve_models() override-first catalog resolver."""
+
+    _ORDER = ("openai/gpt-5", "openai/o3")
+
+    def test_empty_inputs_use_resolver_order(self) -> None:
+        """No env overrides → picks gpt-5 as preferred and o3 as fallback."""
+        available = {"openai/gpt-5", "openai/o3"}
+        preferred, fallback = ra._resolve_models(
+            available=available,
+            catalog_ok=True,
+            env_model="",
+            env_fallback="",
+            order=self._ORDER,
+        )
+        self.assertEqual(preferred, "openai/gpt-5")
+        self.assertEqual(fallback, "openai/o3")
+
+    def test_explicit_primary_overrides_resolver(self) -> None:
+        """GITHUB_MODEL set → use it as preferred regardless of catalog."""
+        available = {"openai/gpt-5", "openai/o3", "openai/other"}
+        preferred, fallback = ra._resolve_models(
+            available=available,
+            catalog_ok=True,
+            env_model="openai/manual",
+            env_fallback="",
+            order=self._ORDER,
+        )
+        self.assertEqual(preferred, "openai/manual")
+
+    def test_explicit_fallback_overrides_resolver(self) -> None:
+        """GITHUB_FALLBACK_MODEL set → use it as fallback, primary still auto-resolved."""
+        available = {"openai/gpt-5", "openai/o3"}
+        preferred, fallback = ra._resolve_models(
+            available=available,
+            catalog_ok=True,
+            env_model="",
+            env_fallback="openai/manual-fb",
+            order=self._ORDER,
+        )
+        self.assertEqual(preferred, "openai/gpt-5")
+        self.assertEqual(fallback, "openai/manual-fb")
+
+    def test_auto_fallback_chooses_o3_when_gpt5_is_primary(self) -> None:
+        """Catalog returns both gpt-5 and o3, no overrides → preferred=gpt-5, fallback=o3."""
+        available = {"openai/gpt-5", "openai/o3"}
+        preferred, fallback = ra._resolve_models(
+            available=available,
+            catalog_ok=True,
+            env_model="",
+            env_fallback="",
+            order=self._ORDER,
+        )
+        self.assertEqual(preferred, "openai/gpt-5")
+        self.assertEqual(fallback, "openai/o3")
+        self.assertNotEqual(preferred, fallback)
+
+    def test_catalog_failure_falls_back_to_order_defaults(self) -> None:
+        """When catalog fails, use order[0] as preferred and order[1] as fallback."""
+        preferred, fallback = ra._resolve_models(
+            available=set(),
+            catalog_ok=False,
+            env_model="",
+            env_fallback="",
+            order=self._ORDER,
+        )
+        self.assertEqual(preferred, "openai/gpt-5")
+        self.assertEqual(fallback, "openai/o3")
+
+
 # ---------------------------------------------------------------------------
 # 4. Syntax validation of generated Python (ast.parse gate)
 # ---------------------------------------------------------------------------