diff --git a/setup/js/build_checkout_manifest.cjs b/setup/js/build_checkout_manifest.cjs new file mode 100644 index 0000000..0e77a57 --- /dev/null +++ b/setup/js/build_checkout_manifest.cjs @@ -0,0 +1,143 @@ +// @ts-check +/// + +require("./shim.cjs"); + +const fs = require("fs"); +const path = require("path"); +const { execFileSync } = require("child_process"); + +const { getErrorMessage } = require("./error_helpers.cjs"); + +function parseManifestEntries(entriesJSON = process.env.GH_AW_CHECKOUT_MANIFEST_ENTRIES || "[]") { + const parsed = JSON.parse(entriesJSON); + if (!Array.isArray(parsed)) { + throw new Error("GH_AW_CHECKOUT_MANIFEST_ENTRIES must be a JSON array"); + } + return parsed; +} + +function readManifestEntriesFromEnv() { + const count = Number.parseInt(process.env.GH_AW_CHECKOUT_MANIFEST_COUNT || "0", 10); + if (!Number.isFinite(count) || count < 0) { + throw new Error("GH_AW_CHECKOUT_MANIFEST_COUNT must be a non-negative integer"); + } + + const entries = []; + for (let i = 0; i < count; i += 1) { + entries.push({ + repository: process.env[`GH_AW_CHECKOUT_REPO_${i}`] || "", + path: process.env[`GH_AW_CHECKOUT_PATH_${i}`] || "", + token: process.env[`GH_AW_CHECKOUT_TOKEN_${i}`] || "", + }); + } + return entries; +} + +function resolveDefaultBranch(repository, checkoutPath, options = {}) { + const workspace = options.workspace || process.env.GITHUB_WORKSPACE || ""; + const runGit = options.runGit || ((args, execOptions = {}) => execFileSync("git", args, { encoding: "utf8", ...execOptions })); + const runGH = + options.runGH || + ((args, execOptions = {}) => + execFileSync("gh", args, { + encoding: "utf8", + env: { ...process.env, ...(execOptions.env || {}) }, + ...execOptions, + })); + let defaultBranch = ""; + + const repoPath = checkoutPath ? path.join(workspace, checkoutPath) : workspace; + if (repoPath && fs.existsSync(path.join(repoPath, ".git"))) { + try { + const output = runGit(["-C", repoPath, "symbolic-ref", "--short", "refs/remotes/origin/HEAD"], { + stdio: ["ignore", "pipe", "pipe"], + }); + defaultBranch = output.trim().replace(/^origin\//, ""); + core.debug(`build_checkout_manifest: git resolved default branch for ${repository}: ${defaultBranch}`); + } catch (error) { + core.debug(`build_checkout_manifest: git default branch lookup failed for ${repository}: ${getErrorMessage(error)}`); + } + } + + if (defaultBranch === "") { + try { + const checkoutToken = options.checkoutToken || ""; + const ghExecOptions = { + stdio: ["ignore", "pipe", "pipe"], + }; + if (checkoutToken !== "") { + ghExecOptions.env = { GH_TOKEN: checkoutToken }; + } + defaultBranch = runGH(["api", `repos/${repository}`, "--jq", ".default_branch"], ghExecOptions).trim(); + core.debug(`build_checkout_manifest: gh api resolved default branch for ${repository}: ${defaultBranch}`); + } catch (error) { + core.debug(`build_checkout_manifest: gh api default branch lookup failed for ${repository}: ${getErrorMessage(error)}`); + } + } + + return defaultBranch; +} + +function buildCheckoutManifest(entries, options = {}) { + const runnerTemp = options.runnerTemp || process.env.RUNNER_TEMP; + if (!runnerTemp) { + throw new Error("RUNNER_TEMP is required to build checkout manifest"); + } + + const runGit = options.runGit; + const runGH = options.runGH; + + const manifestDir = path.join(runnerTemp, "gh-aw"); + fs.mkdirSync(manifestDir, { recursive: true }); + const manifestPath = path.join(manifestDir, "checkout-manifest.json"); + const manifest = {}; + core.info(`checkout-manifest: building manifest for ${entries.length} checkout entries`); + + for (const entry of entries) { + if (!entry || typeof entry !== "object") { + core.debug("checkout-manifest: skipping non-object entry"); + continue; + } + const repository = String(entry.repository || "").trim(); + if (repository === "") { + core.debug("checkout-manifest: skipping entry with empty repository"); + continue; + } + const checkoutPath = String(entry.path || ""); + const defaultBranch = resolveDefaultBranch(repository, checkoutPath, { + workspace: options.workspace, + runGit, + runGH, + checkoutToken: entry.token || "", + }); + manifest[repository.toLowerCase()] = { + repository, + path: checkoutPath, + default_branch: defaultBranch, + }; + core.info(`checkout-manifest: ${repository} -> path=${checkoutPath} default_branch=${defaultBranch || ""}`); + } + + fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf8"); + core.info(`checkout-manifest written to ${manifestPath}`); + return { manifestPath, manifest }; +} + +async function main(options = {}) { + let entries; + if (typeof options.entriesJSON === "string" && options.entriesJSON.trim() !== "") { + entries = parseManifestEntries(options.entriesJSON); + } else { + entries = readManifestEntriesFromEnv(); + } + return buildCheckoutManifest(entries, options); +} + +module.exports = { + buildCheckoutManifest, + main, + parseManifestEntries, + readManifestEntriesFromEnv, + resolveDefaultBranch, +}; diff --git a/setup/js/check_daily_aic_workflow_guardrail.cjs b/setup/js/check_daily_aic_workflow_guardrail.cjs index 380c7e1..34024b5 100644 --- a/setup/js/check_daily_aic_workflow_guardrail.cjs +++ b/setup/js/check_daily_aic_workflow_guardrail.cjs @@ -8,7 +8,7 @@ const path = require("path"); const { calculateDailyAICStats, findJSONLFiles, formatAICCredits, sumAICFromUsageJSONLFiles } = require("./daily_aic_workflow_helpers.cjs"); const { parsePositiveCompactNumber } = require("./numeric_limits.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); -const { createRateLimitAwareGithub } = require("./github_rate_limit_logger.cjs"); +const { createRateLimitAwareGithub, fetchAndLogRateLimit } = require("./github_rate_limit_logger.cjs"); const PRIMARY_GUARDRAIL_ARTIFACT_NAMES = ["usage"]; const DAILY_WORKFLOW_WINDOW_MS = 24 * 60 * 60 * 1000; @@ -313,12 +313,15 @@ async function main() { try { const githubClient = createRateLimitAwareGithub(github); const { owner, repo } = context.repo; + // Capture a before-guardrail rate-limit snapshot and log it to the JSONL + // so consumers can determine the baseline available quota before inspection starts. + const rateLimitStart = await fetchAndLogRateLimit(githubClient, "daily-aic-guardrail-start"); const currentRun = await githubClient.rest.actions.getWorkflowRun({ owner, repo, run_id: context.runId, }); - const rateLimit = await getCoreRateLimitSnapshot(githubClient); + const rateLimit = rateLimitStart ?? (await getCoreRateLimitSnapshot(githubClient)); const workflowID = process.env.GH_AW_WORKFLOW_ID || ""; const workflowName = process.env.GH_AW_WORKFLOW_NAME || workflowID || "workflow"; @@ -460,6 +463,21 @@ async function main() { exceeded: totalAIC > threshold, }); + // Capture an after-guardrail rate-limit snapshot and log it to the JSONL so + // the full cost of the inspection window (workflow-run listing + artifact downloads) + // can be measured. The delta between the before and after snapshots answers + // whether the daily AIC guardrail is too hungry in GitHub API rate limits. + const rateLimitEnd = await fetchAndLogRateLimit(githubClient, "daily-aic-guardrail-end"); + const rateLimitBeforeInspection = rateLimitStart?.remaining ?? rateLimit.remaining; + const rateLimitAfterInspection = rateLimitEnd?.remaining ?? rateLimitBeforeInspection; + logDailyGuardrail("GitHub API rate limit consumed by daily AIC guardrail", { + rateLimitBeforeInspection, + rateLimitAfterInspection, + consumed: Math.max(0, rateLimitBeforeInspection - rateLimitAfterInspection), + limit: rateLimit.limit, + reset: rateLimit.reset, + }); + if (totalAIC <= threshold) { await appendDailyAICSummary(workflowName, actorLogin, threshold, countedRuns, rateLimit, summaryMeta); core.info(`Daily workflow AIC guardrail not exceeded (${totalAIC}/${threshold}).`); diff --git a/setup/js/claude_harness.cjs b/setup/js/claude_harness.cjs index b967ded..1cc6752 100644 --- a/setup/js/claude_harness.cjs +++ b/setup/js/claude_harness.cjs @@ -48,6 +48,7 @@ const { const { emitMissingToolPermissionIssue, hasNoopInSafeOutputs } = require("./safeoutputs_cli.cjs"); const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractDeniedCommands, buildMissingToolPermissionIssuePayload } = require("./permission_denied_helpers.cjs"); const { detectNonRetryableHarnessGuard } = require("./harness_retry_guard.cjs"); +const { MODEL_NOT_SUPPORTED_PATTERN: INVALID_MODEL_ERROR_PATTERN } = require("./detect_agent_errors.cjs"); // Maximum number of retry attempts after the initial run const MAX_RETRIES = 3; @@ -148,6 +149,15 @@ function isNoDeferredMarkerError(output) { return NO_DEFERRED_MARKER_PATTERN.test(output); } +/** + * Determines if the collected output indicates an invalid or unavailable model name. + * @param {string} output - Collected stdout+stderr from the process + * @returns {boolean} + */ +function isInvalidModelError(output) { + return INVALID_MODEL_ERROR_PATTERN.test(output); +} + /** * Determines whether the exit code corresponds to signal-style termination * (SIGKILL=137 / SIGTERM=143), typically from timeout/cancellation. @@ -372,6 +382,7 @@ async function main() { const isAuthenticationFailed = isAuthenticationFailedError(result.output); const isMaxTurns = isMaxTurnsExit(result.output); const isNoDeferredMarker = isNoDeferredMarkerError(result.output); + const isInvalidModel = isInvalidModelError(result.output); const permissionDeniedCount = countPermissionDeniedIssues(result.output); const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(result.output); log( @@ -382,6 +393,7 @@ async function main() { ` isAuthenticationFailedError=${isAuthenticationFailed}` + ` isMaxTurnsExit=${isMaxTurns}` + ` isNoDeferredMarkerError=${isNoDeferredMarker}` + + ` isInvalidModelError=${isInvalidModel}` + ` permissionDeniedCount=${permissionDeniedCount}` + ` hasNumerousPermissionDenied=${hasNumerousPermissionDenied}` + ` hasOutput=${result.hasOutput}` + @@ -411,6 +423,11 @@ async function main() { break; } + if (isInvalidModel) { + log(`attempt ${attempt + 1}: invalid/unsupported model configuration — not retrying (specify a valid engine model name in workflow frontmatter)`); + break; + } + if (hasNumerousPermissionDenied) { const deniedCommands = extractDeniedCommands(result.output); emitMissingToolPermissionIssue({ deniedCommands, logger: log }); @@ -494,6 +511,7 @@ if (typeof module !== "undefined" && module.exports) { isAuthenticationFailedError, isMaxTurnsExit, isNoDeferredMarkerError, + isInvalidModelError, isSignalTerminationExitCode, shouldRetryWithContinue, countPermissionDeniedIssues, diff --git a/setup/js/codex_harness.cjs b/setup/js/codex_harness.cjs index b04855d..a88d402 100644 --- a/setup/js/codex_harness.cjs +++ b/setup/js/codex_harness.cjs @@ -47,6 +47,7 @@ const { const { emitMissingToolPermissionIssue, hasNoopInSafeOutputs } = require("./safeoutputs_cli.cjs"); const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractDeniedCommands, buildMissingToolPermissionIssuePayload } = require("./permission_denied_helpers.cjs"); const { detectNonRetryableHarnessGuard } = require("./harness_retry_guard.cjs"); +const { MODEL_NOT_SUPPORTED_PATTERN: INVALID_MODEL_ERROR_PATTERN } = require("./detect_agent_errors.cjs"); // Maximum number of retry attempts after the initial run const MAX_RETRIES = 3; @@ -120,6 +121,15 @@ function isServerError(output) { return SERVER_ERROR_PATTERN.test(output); } +/** + * Determines if the collected output indicates an invalid or unavailable model name. + * @param {string} output - Collected stdout+stderr from the process + * @returns {boolean} + */ +function isInvalidModelError(output) { + return INVALID_MODEL_ERROR_PATTERN.test(output); +} + /** * Resolve --prompt-file arguments for the Codex run. * Strips the --prompt-file pair from args and appends the file content @@ -402,6 +412,7 @@ async function main() { const isAuthenticationFailed = isAuthenticationFailedError(result.output); const isMissingApiKey = isMissingApiKeyError(result.output); const isServer = isServerError(result.output); + const isInvalidModel = isInvalidModelError(result.output); const permissionDeniedCount = countPermissionDeniedIssues(result.output); const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(result.output); log( @@ -411,6 +422,7 @@ async function main() { ` isAuthenticationFailedError=${isAuthenticationFailed}` + ` isMissingApiKeyError=${isMissingApiKey}` + ` isServerError=${isServer}` + + ` isInvalidModelError=${isInvalidModel}` + ` permissionDeniedCount=${permissionDeniedCount}` + ` hasNumerousPermissionDenied=${hasNumerousPermissionDenied}` + ` hasOutput=${result.hasOutput}` + @@ -445,6 +457,11 @@ async function main() { break; } + if (isInvalidModel) { + log(`attempt ${attempt + 1}: invalid/unsupported model configuration — not retrying (specify a valid engine model name in workflow frontmatter)`); + break; + } + if (hasNumerousPermissionDenied) { const deniedCommands = extractDeniedCommands(result.output); emitMissingToolPermissionIssue({ deniedCommands, logger: log }); @@ -485,6 +502,7 @@ if (typeof module !== "undefined" && module.exports) { isAuthenticationFailedError, isMissingApiKeyError, isServerError, + isInvalidModelError, countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractDeniedCommands, diff --git a/setup/js/detect_agent_errors.cjs b/setup/js/detect_agent_errors.cjs index 074a581..d616228 100644 --- a/setup/js/detect_agent_errors.cjs +++ b/setup/js/detect_agent_errors.cjs @@ -1,7 +1,7 @@ // @ts-check /** - * Detect Copilot CLI errors in the agent stdio log. + * Detect agent engine errors in the agent stdio log. * * Scans the agent stdio log for known error patterns and sets GitHub Actions * output variables for each detected error class: @@ -13,8 +13,9 @@ * - agentic_engine_timeout: The agentic engine process was killed by a * signal (SIGTERM/SIGKILL/SIGINT), typically due to the step * timeout-minutes limit being reached. - * - model_not_supported_error: The requested model is not supported for - * the user's Copilot subscription tier (e.g., Copilot Pro/Education). + * - model_not_supported_error: The configured model is invalid or unsupported + * for the selected engine/account (for example unknown model name, model not + * found, or model unavailable for the plan). * * This replaces the individual bash scripts (detect_inference_access_error.sh, * detect_mcp_policy_error.sh) with a single JavaScript step. @@ -44,11 +45,15 @@ const MCP_POLICY_BLOCKED_PATTERN = /MCP servers were blocked by policy:/; // making it engine-agnostic. const AGENTIC_ENGINE_TIMEOUT_PATTERN = /signal=SIG(?:TERM|KILL|INT)/; -// Pattern: Requested model is not supported for the user's subscription tier. -// This occurs when Copilot Pro/Education users attempt to use a model that is -// not available for their plan. The full error from the Copilot CLI is: -// Execution failed: CAPIError: 400 The requested model is not supported. -const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported/; +// Pattern: Configured model is invalid or unavailable. +// Covers common engine/provider variants: +// - "The requested model is not supported" +// - "invalid model name '...'" +// - "unknown model " +// - "model ... not found" +// - "model ... does not exist" +const MODEL_NOT_SUPPORTED_PATTERN = + /(?:The requested model is not supported|invalid model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|unknown model\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?\s+(?:is\s+)?(?:not found|does not exist|not supported|not available|unavailable))/i; /** * Detect known error patterns in a log string and return detection results. @@ -105,12 +110,14 @@ function main() { process.stderr.write("[detect-agent-errors] Detected timeout: engine process was killed by signal (step timeout-minutes likely exceeded)\n"); } if (results.modelNotSupportedError) { - process.stderr.write("[detect-agent-errors] Detected model-not-supported error: the requested model is unavailable for this subscription tier\n"); + process.stderr.write("[detect-agent-errors] Detected model configuration error: configured model is invalid or unavailable for this engine/account\n"); } writeOutputs(results); } -main(); +if (require.main === module) { + main(); +} module.exports = { detectErrors, INFERENCE_ACCESS_ERROR_PATTERN, MCP_POLICY_BLOCKED_PATTERN, AGENTIC_ENGINE_TIMEOUT_PATTERN, MODEL_NOT_SUPPORTED_PATTERN }; diff --git a/setup/js/github_rate_limit_logger.cjs b/setup/js/github_rate_limit_logger.cjs index a5287d2..fc139b8 100644 --- a/setup/js/github_rate_limit_logger.cjs +++ b/setup/js/github_rate_limit_logger.cjs @@ -123,14 +123,19 @@ function logRateLimitFromResponse(response, operation) { * Use this for a point-in-time snapshot at the start or end of a script, * rather than after every individual API call. * + * Returns the core rate-limit snapshot so callers can use a single API call + * for both logging and in-memory rate-limit tracking. + * * @param {any} github - The github object injected by actions/github-script * @param {string} [operation="fetch"] - Label recorded in each log entry + * @returns {Promise<{remaining:number,limit:number,used:number,reset:string}|null>} + * Core rate-limit data, or null if the call fails or the core resource is absent. */ async function fetchAndLogRateLimit(github, operation = "fetch") { try { const response = await github.rest.rateLimit.get(); const resources = response?.data?.resources; - if (!resources) return; + if (!resources) return null; const timestamp = new Date().toISOString(); for (const [resource, data] of Object.entries(resources)) { @@ -148,8 +153,25 @@ async function fetchAndLogRateLimit(github, operation = "fetch") { }; appendEntry(entry); } + + const coreData = resources.core; + if (!coreData || typeof coreData !== "object") return null; + const remaining = Number(coreData.remaining); + const limit = Number(coreData.limit); + const used = Number(coreData.used); + const resetSeconds = Number(coreData.reset); + if (!Number.isFinite(remaining) || !Number.isFinite(limit) || !Number.isFinite(used) || !Number.isFinite(resetSeconds)) { + return null; + } + return { + remaining, + limit, + used, + reset: new Date(resetSeconds * 1000).toISOString(), + }; } catch (err) { core.warning(`github_rate_limit_logger: fetchAndLogRateLimit failed: ${getErrorMessage(err)}`); + return null; } } diff --git a/setup/md/model_not_supported_error.md b/setup/md/model_not_supported_error.md index 44a7d13..e54070a 100644 --- a/setup/md/model_not_supported_error.md +++ b/setup/md/model_not_supported_error.md @@ -1,12 +1,12 @@ > [!WARNING] -> **Model Not Supported**: The Copilot CLI failed because the requested model is not available for your subscription tier. This typically affects Copilot Pro and Education users. +> **Invalid or Unsupported Model**: The agent failed because the configured model name is invalid, unknown, or unavailable for this engine/account. This is a **configuration issue**, not a transient error — retrying will not help.
How to fix this -Specify a model that is supported by your subscription in the workflow frontmatter: +Specify a valid model for the selected engine in the workflow frontmatter: ```yaml --- @@ -15,6 +15,6 @@ model: gpt-5-mini --- ``` -To find the models available for your account, check your [Copilot settings](https://github.com/settings/copilot) or refer to the [supported models documentation](https://docs.github.com/en/copilot/using-github-copilot/using-github-copilot-in-the-command-line#supported-models). +To find valid models, check your engine/provider documentation (for Copilot see [supported models](https://docs.github.com/en/copilot/using-github-copilot/using-github-copilot-in-the-command-line#supported-models)).