Improve commit message suggestion workflow (#2000)

Stephan202 · web-flow · commit 4f229570e3ec · 2025-11-26T10:14:26.000+01:00
Summary of changes:
- Defer `git` command execution to the agent, and guide it to execute
  only relevant commands.
- Suggest a better command to retrieve past commit upgrade messages.
- Enable limited network access, so that Github milestone and release
  candidate versions can be retrieved.
- Improve the instructions for the inclusion of custom release note
  links.
- Attach the suggested commit message as a workflow artifact, and 
  provide a script to retrieve it. This enables command line-based
  development iterations.
diff --git a/.github/workflows/suggest-commit-message.yml b/.github/workflows/suggest-commit-message.yml
@@ -17,6 +17,12 @@ permissions:
 concurrency:
   group: suggest-commit-message-${{ github.event.pull_request.number || github.event.inputs.pr_number }}
   cancel-in-progress: true
+env:
+  ALLOWED_ENDPOINTS: >
+    api.github.com:443
+    api.openai.com:443
+    github.com:443
+    registry.npmjs.org:443
 jobs:
   suggest:
     permissions:
@@ -29,14 +35,13 @@ jobs:
         uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
         with:
           # We can't disable `sudo`, as `openai/codex-action` unconditionally
-          # invokes `sudo`. That step does disable `sudo` for itself and
-          # subsequent steps.
-          # XXX: Consider splitting this workflow into two jobs, with
-          # `openai/codex-action` being the first step of the second job.
+          # invokes `sudo`, even with `safety-strategy: unsafe` and
+          # `sandbox: danger-full-access`.
+          # XXX: Consider splitting this workflow into three jobs, with
+          # `openai/codex-action` being the sole step of the second job.
           disable-sudo-and-containers: false
-          # XXX: Change to `egress-policy: block` once we better understand
-          # whether Codex attempts to access arbitrary URLs.
-          egress-policy: audit
+          egress-policy: block
+          allowed-endpoints: ${{ env.ALLOWED_ENDPOINTS }}
       - name: Resolve pull request metadata
         id: pr-details
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
@@ -82,24 +87,8 @@ jobs:
           HEAD_SHA: ${{ steps.pr-details.outputs.headSha }}
         with:
           script: |
-            const { execFileSync } = require('child_process');
             const fs = require('fs');
 
-            const git = (args, limit) => {
-              const output = execFileSync('git', args, { encoding: 'utf8' }).trim();
-              if (!limit || !output) {
-                return output;
-              }
-
-              const lines = output.split(/\r?\n/);
-              if (lines.length <= limit) {
-                return output;
-              }
-
-              const truncated = lines.slice(0, limit).join('\n');
-              return `${truncated}\n... (${limit} of ${lines.length} lines shown)`;
-            };
-
             const env = process.env;
             const repository = env.REPOSITORY;
             const prNumber = env.PR_NUMBER;
@@ -111,19 +100,30 @@ jobs:
             const headRef = env.HEAD_REF;
             const headSha = env.HEAD_SHA;
 
-            const diffStat = git(['diff', '--name-status', `${baseSha}...${headSha}`]) || '<no changed files>';
-            const diffExcerpt = git(['diff', '--unified=3', `${baseSha}...${headSha}`], 500) || '<no diff>';
-            const nonUpgradeCommits =
-              git(['log', '--grep', '^Upgrade', '--invert-grep', '--pretty=format:%h %B%n---', '-n', '50', baseSha]) ||
-              '<no non-upgrade commits found>';
-            const upgradeCommits =
-              git(['log', '--grep', '^Upgrade', '--pretty=format:%h %B%n---', '-n', '150', baseSha]) ||
-              '<no upgrade commits found>';
-
             const cleanedBody = (body || '').trim() || '<no pull request description>';
 
+            // Determine whether this is an upgrade PR.
+            const upgradeMatch = title?.match(/^Upgrade (.+?) \S+ -> \S+/);
+            const upgradeLibrary = upgradeMatch ? upgradeMatch[1] : null;
+
+            // Extract domain names from list of endpoints to which Harden-Runner will allow access.
+            const allowedDomains = process.env.ALLOWED_ENDPOINTS.split(/\s+/).map(line => line.split(':')[0]).filter(Boolean);
+
             const instructions = `
-            You are an experienced maintainer helping to craft the squash commit message for PR #${prNumber} in the ${repository} repository.
+            You are an experienced maintainer helping to craft the squash commit message for a GitHub pull request.
+
+            Pull request metadata:
+            - Repository: ${repository}
+            - Number: ${prNumber}
+            - Title: ${title}
+            - Author: ${author}
+            - Base branch: ${baseRef} (${baseSha})
+            - Head branch: ${headRef} (${headSha})
+
+            Pull request description:
+            \`\`\`
+            ${cleanedBody}
+            \`\`\`
 
             Requirements:
             1. Write the summary line in the imperative mood. Try not to exceed 80 characters.
@@ -137,19 +137,43 @@ jobs:
             9. Never split URLs across multiple lines, even if they exceed 72 characters.
             10. If the pull request description already contains a suitable commit message, prefer using that as-is.
 
+            To help you craft an appropriate commit message, execute the following commands to gather context:
+
+            1. Get the changed files:
+            \`\`\`
+            git diff --name-status ${baseSha}...${headSha}
+            \`\`\`
+
+            2. Get a diff excerpt (first 500 lines):
+            \`\`\`
+            git diff ${baseSha}...${headSha} | head -500
+            \`\`\`
+
+            ${upgradeLibrary ? `3. Since this appears to be an upgrade PR for ${upgradeLibrary}, collect relevant past upgrade commit messages, and consider the general style of other recent upgrade commit messages:
+            \`\`\`
+            git log -P -i --grep '^Upgrade \\Q${upgradeLibrary}\\E' --pretty='format:%h %B%n---' -n 20 ${baseSha}
+            git log -P -i --grep '^Upgrade (?!\\Q${upgradeLibrary}\\E)' --pretty='format:%h %B%n---' -n 150 ${baseSha}
+            \`\`\`
+
+            4. If this is a GitHub-hosted library, collect milestones and release candidates that may not be included in the changelog:
+            \`\`\`
+            curl -s "https://api.github.com/repos/{owner}/{repo}/releases?per_page=100" | jq -r '.[].tag_name' | sort -h | tail -n 50
+            \`\`\`` : `3. Get examples of recent non-upgrade commits:
+            \`\`\`
+            git log --grep '^Upgrade' --invert-grep --pretty='format:%h %B%n---' -n 50 ${baseSha}
+            \`\`\``}
+
             Some further guidelines to help you craft good upgrade commit messages:
             - Unless highly salient, don't summarize code changes made as part of the upgrade.
             - Don't bother linking to anchors within changelogs or release notes; just link to the main page.
-            - For GitHub-hosted projects, always link to all relevant GitHub release pages, including those for intermediate versions.
-              - This includes milestones and release candidates; if necessary, use the GitHub API to identify these.
-              - Libraries that often use milestone and release candidates include, but are not limited to:
-                - Jackson
-                - JUnit
-                - Micrometer
-                - Project Reactor
-                - Spring Framework
-                - Spring Boot
-                - Spring Security
+            - For GitHub-hosted projects, always link to all relevant GitHub release pages, including those for milestones, release candidates and other intermediate versions. This is especially important for major and minor version upgrades of the following libraries:
+              - Jackson
+              - JUnit
+              - Micrometer
+              - Project Reactor
+              - Spring Framework
+              - Spring Boot
+              - Spring Security
             - For GitHub-hosted projects, always link to the full diff between versions.
             - Enumerate links in the following order:
               1. First, link to custom release note documents.
@@ -159,12 +183,15 @@ jobs:
             - When the Maven \u0060version.error-prone-orig\u0060 property is changed, this upgrades both Error Prone and Picnic's Error Prone fork. In this case:
               - Make sure that the commit message includes a diff URL for the latter.
               - Don't explicitly mention that \u0060version.error-prone-orig\u0060 got changed; just focus on the fact that Error Prone is being upgraded.
-            - If the example upgrade commits shown below don't include at least one upgrade of the same dependency being upgraded in this pull request, check the full Git history to find relevant past upgrade commit messages to mimic.
             - For major and minor version upgrades, check past dependency upgrade commit messages to infer documentation, blog or wiki URLs to which to link. Do this for at least the following libraries:
-              - Jackson
-              - Spring Framework
-              - Spring Boot
-              - Spring Security
+              - Jackson: https://github.com/FasterXML/jackson/wiki/Jackson-Release-{version}
+              - Spring Framework: https://github.com/spring-projects/spring-framework/wiki/Spring-Framework-{version}-Release-Notes
+              - Spring Boot: https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-{version}-Release-Notes
+              - Spring Security: https://docs.spring.io/spring-security/reference/{version}/whats-new.html
+            - If you really can't find relevant URLs to reference, and there's nothing else to say, it's acceptable to have a commit message that only consists of the summary line.
+
+            Note that your network access is limited to the following domains; don't attempt \`curl\` or \`wget\` commands to other hosts:
+            ${allowedDomains.map(domain => `- ${domain}`).join('\n')}
 
             Return a JSON object with the following shape:
             {
@@ -173,38 +200,6 @@ jobs:
             }
 
             Ensure the JSON is valid. Do not include additional commentary outside the JSON structure.
-
-            Pull request metadata:
-            - Number: ${prNumber}
-            - Title: ${title}
-            - Author: ${author}
-            - Base branch: ${baseRef} (${baseSha})
-            - Head branch: ${headRef} (${headSha})
-
-            Pull request description:
-            \u0060\u0060\u0060
-            ${cleanedBody}
-            \u0060\u0060\u0060
-
-            Changed files (\u0060git diff --name-status ${baseSha}...${headSha}\u0060):
-            \u0060\u0060\u0060
-            ${diffStat}
-            \u0060\u0060\u0060
-
-            Diff excerpt (\u0060git diff --unified=3 ${baseSha}...${headSha}\u0060, truncated to 500 lines if necessary):
-            \u0060\u0060\u0060
-            ${diffExcerpt}
-            \u0060\u0060\u0060
-
-            Recent non-upgrade commits examples (\u0060git log --grep '^Upgrade' --invert-grep --pretty='format:%h %B%n---' -n 50\u0060):
-            \u0060\u0060\u0060
-            ${nonUpgradeCommits}
-            \u0060\u0060\u0060
-
-            Recent upgrade commit examples (\u0060git log --grep '^Upgrade' --pretty='format:%h %B%n---' -n 150\u0060):
-            \u0060\u0060\u0060
-            ${upgradeCommits}
-            \u0060\u0060\u0060
             `;
 
             const promptPath = '/tmp/codex-prompt-suggest-commit-message.md';
@@ -213,11 +208,12 @@ jobs:
         id: codex
         uses: openai/codex-action@086169432f1d2ab2f4057540b1754d550f6a1189 # v1.4
         with:
-          # XXX: Consider whether to set `safety-strategy: read-only`. In some
-          # cases the agent may be able to suggest a better commit message by
-          # following links or otherwise looking up information online. See
-          # also the `egress-policy` discussion further up.
-          sandbox: read-only
+          # XXX: We're using `safety-strategy: unsafe` and
+          # `sandbox: danger-full-access` so that the agent is able to access
+          # the network and look up e.g. GitHub release tags. Some amount of
+          # safety is provided by the Harden-Runner step further up.
+          safety-strategy: unsafe
+          sandbox: danger-full-access
           openai-api-key: ${{ secrets.OPENAI_API_KEY }}
           prompt-file: /tmp/codex-prompt-suggest-commit-message.md
           output-schema: |
@@ -244,16 +240,22 @@ jobs:
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
+            const fs = require('fs');
+
             const prNumber = process.env.PR_NUMBER;
             const codexResult = JSON.parse(process.env.CODEX_RESULT);
 
             const summary = codexResult.summary.trim();
             const body = codexResult.body.trim();
             const commitMessage = body ? `${summary}\n\n${body}` : summary;
 
+            // Write the commit message to a file, so that the next step can
+            // attach it as a workflow artifact for debug purposes.
+            fs.writeFileSync('/tmp/suggested-commit-message.txt', commitMessage, { encoding: 'utf8' });
+
             // The comment to be upserted includes a hidden marker to identify it.
             const marker = '<!-- codex-suggested-commit-message -->';
-            const commentBody = `Suggested commit message:\n${marker}\n\n\u0060\u0060\u0060\n${commitMessage}\n\u0060\u0060\u0060\n`;
+            const commentBody = `Suggested commit message:\n${marker}\n\n\`\`\`\n${commitMessage}\n\`\`\`\n`;
 
             const comments = await github.paginate(github.rest.issues.listComments, {
               owner: context.repo.owner,
@@ -311,3 +313,8 @@ jobs:
               body: commentBody,
             });
             core.info(`Updated comment ${existing.id} by ${originalCommenter}.`);
+      - name: Upload suggested commit message
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: suggested-commit-message
+          path: /tmp/suggested-commit-message.txt
diff --git a/suggest-commit-message.sh b/suggest-commit-message.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Upserts a suggested commit message comment on the specified pull request,
+# based on the workflow definition present on the specified branch. The
+# generated commit message is also written to stdout.
+
+set -e -u -o pipefail
+
+workflow='suggest-commit-message.yml'
+branch="${1:?Specify a branch}"
+pr_number="${2:?Specify a PR number}"
+
+gh workflow run "${workflow}" --ref "${branch}" -f pr_number="${pr_number}"
+
+# The new run may not start immediately, so we wait a bit.
+sleep 30
+
+run_id=$(
+  gh run list --workflow="${workflow}" --event workflow_dispatch --limit 1 --json databaseId \
+    | jq -r '.[].databaseId'
+)
+
+gh run watch "${run_id}"
+
+tmp_dir="$(mktemp -d)"
+trap 'rm -rf -- "${tmp_dir}"' INT TERM HUP EXIT
+
+gh run download -n suggested-commit-message --dir "${tmp_dir}" "${run_id}"
+cat "${tmp_dir}/suggested-commit-message.txt"
