feat: warn on classic PAT + COPILOT_MODEL incompatibility (Copilot CLI 1.0.21+) (#1907)

* Initial plan

* feat: warn when classic PAT is used with COPILOT_MODEL

Agent-Logs-Url: https://github.com/github/gh-aw-firewall/sessions/99b813df-d6d8-41a5-9288-a8516b85733d

* fix: clarify variable name and test assertion for PAT warning

* Update src/cli.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/cli.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update docs/environment.md

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: Landon Cox <landon.cox@microsoft.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

docs/environment.md

@@ -88,6 +88,32 @@ sudo -E awf --allow-domains github.com 'copilot --prompt "..."'
 - Working with untrusted code
 - In production/CI environments
 
+## `COPILOT_GITHUB_TOKEN` and Classic PAT Compatibility
+
+When `COPILOT_GITHUB_TOKEN` is set in the host environment, AWF injects it into the agent container so the Copilot CLI can authenticate against the GitHub Copilot API.
+
+### ⚠️ Classic PAT + `COPILOT_MODEL` Incompatibility (Copilot CLI 1.0.21+)
+
+Copilot CLI 1.0.21 introduced a startup model validation step: when `COPILOT_MODEL` is set, the CLI calls `GET /models` before executing any task. **This endpoint does not accept classic PATs** (`ghp_*` tokens), causing the agent to fail at startup with exit code 1 — before any useful work begins.
+
+**Affected combination:**
+- `COPILOT_GITHUB_TOKEN` is a classic PAT (prefixed with `ghp_`)
+- `COPILOT_MODEL` is set in the agent environment (e.g., via `--env COPILOT_MODEL=...`, `--env-file`, or `--env-all`)
+
+**Unaffected:** Workflows that do not set `COPILOT_MODEL` are not affected — the `/models` validation is only triggered when `COPILOT_MODEL` is set.
+
+**AWF detects this combination at startup** and emits a `[WARN]` message:
+```
+⚠️  COPILOT_MODEL is set with a classic PAT (ghp_* token)
+   Copilot CLI 1.0.21+ validates COPILOT_MODEL via GET /models at startup.
+   Classic PATs are rejected by this endpoint — the agent will likely fail with exit code 1.
+   Use a fine-grained PAT or OAuth token, or unset COPILOT_MODEL to skip model validation.
+```
+
+**Remediation options:**
+1. Replace the classic PAT with a **fine-grained PAT** or **OAuth token** (these are accepted by the `/models` endpoint).
+2. Remove `COPILOT_MODEL` from the agent environment to skip model validation entirely.
+
 ## Internal Environment Variables
 
 The following environment variables are set internally by the firewall and used by container scripts:

src/cli.test.ts

@@ -1,5 +1,5 @@
 import { Command } from 'commander';
-import { parseEnvironmentVariables, parseDomains, parseDomainsFile, escapeShellArg, joinShellArgs, parseVolumeMounts, isValidIPv4, isValidIPv6, parseDnsServers, parseDnsOverHttps, validateAgentImage, isAgentImagePreset, AGENT_IMAGE_PRESETS, processAgentImageOption, processLocalhostKeyword, validateSkipPullWithBuildLocal, validateAllowHostPorts, validateAllowHostServicePorts, applyHostServicePortsConfig, parseMemoryLimit, validateFormat, validateApiProxyConfig, buildRateLimitConfig, validateRateLimitFlags, hasRateLimitOptions, collectRulesetFile, validateApiTargetInAllowedDomains, DEFAULT_OPENAI_API_TARGET, DEFAULT_ANTHROPIC_API_TARGET, DEFAULT_COPILOT_API_TARGET, DEFAULT_GEMINI_API_TARGET, emitApiProxyTargetWarnings, emitCliProxyStatusLogs, formatItem, program, parseAgentTimeout, applyAgentTimeout, handlePredownloadAction, resolveApiTargetsToAllowedDomains, extractGhesDomainsFromEngineApiTarget, extractGhecDomainsFromServerUrl } from './cli';
+import { parseEnvironmentVariables, parseDomains, parseDomainsFile, escapeShellArg, joinShellArgs, parseVolumeMounts, isValidIPv4, isValidIPv6, parseDnsServers, parseDnsOverHttps, validateAgentImage, isAgentImagePreset, AGENT_IMAGE_PRESETS, processAgentImageOption, processLocalhostKeyword, validateSkipPullWithBuildLocal, validateAllowHostPorts, validateAllowHostServicePorts, applyHostServicePortsConfig, parseMemoryLimit, validateFormat, validateApiProxyConfig, buildRateLimitConfig, validateRateLimitFlags, hasRateLimitOptions, collectRulesetFile, validateApiTargetInAllowedDomains, DEFAULT_OPENAI_API_TARGET, DEFAULT_ANTHROPIC_API_TARGET, DEFAULT_COPILOT_API_TARGET, DEFAULT_GEMINI_API_TARGET, emitApiProxyTargetWarnings, emitCliProxyStatusLogs, warnClassicPATWithCopilotModel, formatItem, program, parseAgentTimeout, applyAgentTimeout, handlePredownloadAction, resolveApiTargetsToAllowedDomains, extractGhesDomainsFromEngineApiTarget, extractGhecDomainsFromServerUrl } from './cli';
 import { redactSecrets } from './redact-secrets';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -2112,6 +2112,46 @@ describe('cli', () => {
     });
   });
 
+  describe('warnClassicPATWithCopilotModel', () => {
+    it('should emit warnings when classic PAT and COPILOT_MODEL are both set', () => {
+      const warns: string[] = [];
+      warnClassicPATWithCopilotModel(true, true, (msg) => warns.push(msg));
+      expect(warns.length).toBeGreaterThan(0);
+      expect(warns[0]).toContain('COPILOT_MODEL');
+      expect(warns.some(w => w.includes('classic PAT'))).toBe(true);
+    });
+
+    it('should not warn when token is not a classic PAT', () => {
+      const warns: string[] = [];
+      warnClassicPATWithCopilotModel(false, true, (msg) => warns.push(msg));
+      expect(warns).toHaveLength(0);
+    });
+
+    it('should not warn when COPILOT_MODEL is not set', () => {
+      const warns: string[] = [];
+      warnClassicPATWithCopilotModel(true, false, (msg) => warns.push(msg));
+      expect(warns).toHaveLength(0);
+    });
+
+    it('should not warn when neither condition holds', () => {
+      const warns: string[] = [];
+      warnClassicPATWithCopilotModel(false, false, (msg) => warns.push(msg));
+      expect(warns).toHaveLength(0);
+    });
+
+    it('should mention /models endpoint in warning', () => {
+      const warns: string[] = [];
+      warnClassicPATWithCopilotModel(true, true, (msg) => warns.push(msg));
+      expect(warns.some(w => w.includes('/models'))).toBe(true);
+    });
+
+    it('should mention exit code 1 in warning', () => {
+      const warns: string[] = [];
+      warnClassicPATWithCopilotModel(true, true, (msg) => warns.push(msg));
+      expect(warns.some(w => w.includes('exit code 1'))).toBe(true);
+    });
+  });
+
   describe('resolveApiTargetsToAllowedDomains', () => {
     it('should add copilot-api-target option to allowed domains', () => {
       const domains: string[] = ['github.com'];

src/cli.ts

@@ -420,6 +420,30 @@ export function emitCliProxyStatusLogs(
   }
 }
 
+/**
+ * Warns when a classic GitHub PAT (ghp_* prefix) is used alongside COPILOT_MODEL.
+ * Copilot CLI 1.0.21+ performs a GET /models validation at startup when COPILOT_MODEL
+ * is set. This endpoint rejects classic PATs, causing the agent to fail with exit code 1
+ * before any useful work begins.
+ * Accepts booleans (not actual tokens/values) to prevent sensitive data from flowing
+ * through to log output (CodeQL: clear-text logging of sensitive information).
+ * @param isClassicPAT - Whether COPILOT_GITHUB_TOKEN starts with 'ghp_' (classic PAT)
+ * @param hasCopilotModel - Whether COPILOT_MODEL is set in the agent environment
+ * @param warn - Function to emit a warning message
+ */
+export function warnClassicPATWithCopilotModel(
+  isClassicPAT: boolean,
+  hasCopilotModel: boolean,
+  warn: (msg: string) => void,
+): void {
+  if (!isClassicPAT || !hasCopilotModel) return;
+
+  warn('⚠️  COPILOT_MODEL is set with a classic PAT (ghp_* token)');
+  warn('   Copilot CLI 1.0.21+ validates COPILOT_MODEL via GET /models at startup.');
+  warn('   Classic PATs are rejected by this endpoint — the agent will likely fail with exit code 1.');
+  warn('   Use a fine-grained PAT or OAuth token, or unset COPILOT_MODEL to skip model validation.');
+}
+
 /**
  * Extracts GHEC domains from GITHUB_SERVER_URL and GITHUB_API_URL environment variables.
  * When GITHUB_SERVER_URL points to a GHEC tenant (*.ghe.com), returns the tenant hostname,
@@ -1935,6 +1959,39 @@ program
     // Log CLI proxy status
     emitCliProxyStatusLogs(config, logger.info.bind(logger), logger.warn.bind(logger));
 
+    // Warn if a classic PAT is combined with COPILOT_MODEL (Copilot CLI 1.0.21+ incompatibility)
+    const hasCopilotModelInEnvFiles = (envFile: unknown): boolean => {
+      const envFiles = Array.isArray(envFile) ? envFile : envFile ? [envFile] : [];
+      for (const candidate of envFiles) {
+        if (typeof candidate !== 'string' || candidate.trim() === '') continue;
+        try {
+          const envFilePath = path.isAbsolute(candidate) ? candidate : path.resolve(process.cwd(), candidate);
+          const envFileContents = fs.readFileSync(envFilePath, 'utf8');
+          for (const line of envFileContents.split(/\r?\n/)) {
+            const trimmedLine = line.trim();
+            if (!trimmedLine || trimmedLine.startsWith('#')) continue;
+            if (/^(?:export\s+)?COPILOT_MODEL\s*=/.test(trimmedLine)) {
+              return true;
+            }
+          }
+        } catch {
+          // Ignore unreadable env files here; this check is only for a pre-flight warning.
+        }
+      }
+      return false;
+    };
+
+    // Warn if a classic PAT is combined with COPILOT_MODEL (Copilot CLI 1.0.21+ incompatibility)
+    // Check if COPILOT_MODEL is set via --env/-e flags, host env (when --env-all is active), or --env-file
+    const copilotModelFromFlags = !!(additionalEnv['COPILOT_MODEL']);
+    const copilotModelInHostEnv = !!(config.envAll && process.env.COPILOT_MODEL);
+    const copilotModelInEnvFile = hasCopilotModelInEnvFiles((config as { envFile?: unknown }).envFile);
+    warnClassicPATWithCopilotModel(
+      config.copilotGithubToken?.startsWith('ghp_') ?? false,
+      copilotModelFromFlags || copilotModelInHostEnv || copilotModelInEnvFile,
+      logger.warn.bind(logger)
+    );
+
     // Log config with redacted secrets - remove API keys entirely
     // to prevent sensitive data from flowing to logger (CodeQL sensitive data logging)
     const redactedConfig: Record<string, unknown> = {};