Skip to content

Commit beddbe7

Browse files
sartajclaudegithub-actions[bot]
authored
Add nightly real world test of govbot (#12)
Adds a VHS tape and GitHub Actions workflow that runs nightly to verify the production govbot binary works end-to-end: install → wizard → clone. https://claude.ai/code/session_01NTKxu7JZzaTUyDSLV71hB6 --------- Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent e77d380 commit beddbe7

File tree

8 files changed

+281
-80
lines changed

8 files changed

+281
-80
lines changed

‎.github/workflows/synthetic-test.yml‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
name: Nightly Real World End To End Test
2+
3+
on:
4+
schedule:
5+
- cron: '0 6 * * *' # 6 AM UTC daily
6+
workflow_run:
7+
workflows: ["Govbot Nightly"]
8+
types: [completed]
9+
branches: [main]
10+
pull_request:
11+
paths:
12+
- .github/workflows/synthetic-test.yml
13+
- actions/govbot/tapes/nightly/synthetic-test.tape
14+
- actions/govbot/src/**
15+
workflow_dispatch:
16+
17+
jobs:
18+
synthetic-test:
19+
name: Govbot Wizard & Pipeline
20+
runs-on: ubuntu-latest
21+
timeout-minutes: 15
22+
permissions:
23+
contents: write
24+
if: >-
25+
github.event_name != 'workflow_run' ||
26+
github.event.workflow_run.conclusion == 'success'
27+
28+
steps:
29+
- name: Checkout
30+
uses: actions/checkout@v4
31+
with:
32+
ref: ${{ github.head_ref || '' }}
33+
34+
- name: Setup Rust toolchain
35+
uses: dtolnay/rust-toolchain@stable
36+
37+
- name: Cache cargo
38+
uses: actions/cache@v4
39+
with:
40+
path: |
41+
~/.cargo/bin/
42+
~/.cargo/registry/index/
43+
~/.cargo/registry/cache/
44+
~/.cargo/git/db/
45+
actions/govbot/target/
46+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
47+
restore-keys: |
48+
${{ runner.os }}-cargo-
49+
50+
- name: Build govbot
51+
run: |
52+
cd actions/govbot
53+
cargo build --release
54+
mkdir -p "$HOME/.govbot/bin"
55+
cp target/release/govbot "$HOME/.govbot/bin/govbot"
56+
echo "$HOME/.govbot/bin" >> $GITHUB_PATH
57+
58+
- name: Install VHS dependencies
59+
run: |
60+
sudo apt-get update
61+
sudo apt-get install -y ffmpeg
62+
TTYD_VERSION="1.7.7"
63+
curl -fsSL "https://github.com/tsl0922/ttyd/releases/download/${TTYD_VERSION}/ttyd.x86_64" -o ttyd
64+
chmod +x ttyd
65+
sudo mv ttyd /usr/local/bin/
66+
67+
- name: Install VHS
68+
run: |
69+
VHS_VERSION="0.9.0"
70+
curl -fsSL "https://github.com/charmbracelet/vhs/releases/download/v${VHS_VERSION}/vhs_${VHS_VERSION}_amd64.deb" -o vhs.deb
71+
sudo dpkg -i vhs.deb
72+
rm vhs.deb
73+
74+
- name: Run end-to-end test via VHS (with retry)
75+
run: |
76+
DIAG=/tmp/vhs-diagnostic.txt
77+
run_test() {
78+
rm -rf /tmp/govbot-test /tmp/govbot-synthetic-test.gif
79+
vhs actions/govbot/tapes/nightly/synthetic-test.tape 2>&1 | tee /tmp/vhs-output.txt
80+
local vhs_rc=${PIPESTATUS[0]}
81+
{
82+
echo "VHS exit code: $vhs_rc"
83+
echo "--- VHS output ---"
84+
cat /tmp/vhs-output.txt 2>/dev/null
85+
echo "--- /tmp/govbot-test contents ---"
86+
ls -la /tmp/govbot-test/ 2>/dev/null || echo " (dir missing)"
87+
echo "--- GIF exists? ---"
88+
ls -la /tmp/govbot-synthetic-test.gif 2>/dev/null || echo " (no GIF)"
89+
echo "--- govbot.yml? ---"
90+
cat /tmp/govbot-test/govbot.yml 2>/dev/null || echo " (no govbot.yml)"
91+
echo "--- pipeline log? ---"
92+
cat /tmp/pipeline.log 2>/dev/null || echo " (no pipeline.log)"
93+
} >> "$DIAG"
94+
[ "$vhs_rc" -eq 0 ] || return 1
95+
test -f /tmp/govbot-synthetic-test.gif || { echo "FAIL: no GIF" >> "$DIAG"; return 1; }
96+
test -f /tmp/govbot-test/govbot.yml || { echo "FAIL: no govbot.yml" >> "$DIAG"; return 1; }
97+
test -d /tmp/govbot-test/.govbot/repos/ || { echo "FAIL: no repos dir" >> "$DIAG"; return 1; }
98+
test -d /tmp/govbot-test/docs/ || { echo "FAIL: no docs dir" >> "$DIAG"; return 1; }
99+
}
100+
101+
if run_test; then
102+
echo "End-to-end test passed on attempt 1"
103+
else
104+
echo "::warning::Attempt 1 failed, retrying..."
105+
sleep 5
106+
if run_test; then
107+
echo "End-to-end test passed on attempt 2"
108+
else
109+
echo "::error::End-to-end test failed after 2 attempts"
110+
exit 1
111+
fi
112+
fi
113+
114+
echo "=== govbot.yml ==="
115+
cat /tmp/govbot-test/govbot.yml
116+
echo "=== Cloned repos ==="
117+
ls -la /tmp/govbot-test/.govbot/repos/
118+
echo "=== Build output ==="
119+
ls -la /tmp/govbot-test/docs/
120+
121+
- name: Upload GIF to synthetic-test release
122+
if: always()
123+
env:
124+
GH_TOKEN: ${{ github.token }}
125+
run: |
126+
if [ -f /tmp/govbot-synthetic-test.gif ]; then
127+
gh release create synthetic-test --title "Synthetic Test Recording" --notes "Latest end-to-end test recording. Updated automatically." 2>/dev/null || true
128+
gh release upload synthetic-test /tmp/govbot-synthetic-test.gif --clobber || true
129+
fi
130+
131+
- name: Job summary
132+
if: always()
133+
run: |
134+
echo "## Nightly Real World End To End Test Results" >> $GITHUB_STEP_SUMMARY
135+
echo "" >> $GITHUB_STEP_SUMMARY
136+
echo "### Recording" >> $GITHUB_STEP_SUMMARY
137+
echo "![Synthetic Test](https://github.com/chihacknight/govbot/releases/download/synthetic-test/govbot-synthetic-test.gif)" >> $GITHUB_STEP_SUMMARY
138+
echo "" >> $GITHUB_STEP_SUMMARY
139+
if [ -f /tmp/govbot-test/govbot.yml ]; then
140+
echo "### govbot.yml" >> $GITHUB_STEP_SUMMARY
141+
echo '```yaml' >> $GITHUB_STEP_SUMMARY
142+
cat /tmp/govbot-test/govbot.yml >> $GITHUB_STEP_SUMMARY
143+
echo '```' >> $GITHUB_STEP_SUMMARY
144+
else
145+
echo "> govbot.yml was not generated" >> $GITHUB_STEP_SUMMARY
146+
fi
147+
echo "" >> $GITHUB_STEP_SUMMARY
148+
echo "### Cloned repos" >> $GITHUB_STEP_SUMMARY
149+
if [ -d /tmp/govbot-test/.govbot/repos ]; then
150+
echo '```' >> $GITHUB_STEP_SUMMARY
151+
ls /tmp/govbot-test/.govbot/repos/ >> $GITHUB_STEP_SUMMARY
152+
echo '```' >> $GITHUB_STEP_SUMMARY
153+
else
154+
echo "> No repos were cloned" >> $GITHUB_STEP_SUMMARY
155+
fi
156+
echo "" >> $GITHUB_STEP_SUMMARY
157+
echo "### Build output" >> $GITHUB_STEP_SUMMARY
158+
if [ -d /tmp/govbot-test/docs ]; then
159+
echo '```' >> $GITHUB_STEP_SUMMARY
160+
ls /tmp/govbot-test/docs/ >> $GITHUB_STEP_SUMMARY
161+
echo '```' >> $GITHUB_STEP_SUMMARY
162+
else
163+
echo "> No build output was generated" >> $GITHUB_STEP_SUMMARY
164+
fi
165+
echo "" >> $GITHUB_STEP_SUMMARY
166+
if [ -f /tmp/vhs-diagnostic.txt ]; then
167+
echo "### Diagnostics" >> $GITHUB_STEP_SUMMARY
168+
echo '```' >> $GITHUB_STEP_SUMMARY
169+
tail -100 /tmp/vhs-diagnostic.txt >> $GITHUB_STEP_SUMMARY
170+
echo '```' >> $GITHUB_STEP_SUMMARY
171+
fi
172+
173+
- name: Upload recording
174+
if: always()
175+
uses: actions/upload-artifact@v4
176+
with:
177+
name: synthetic-test-recording
178+
path: /tmp/govbot-synthetic-test.gif
179+
retention-days: 7
180+
181+
- name: Upload diagnostics
182+
if: always()
183+
uses: actions/upload-artifact@v4
184+
with:
185+
name: vhs-diagnostics
186+
path: |
187+
/tmp/vhs-diagnostic.txt
188+
/tmp/vhs-output.txt
189+
retention-days: 7
190+
if-no-files-found: ignore

‎.github/workflows/terminal-screenshots.yml‎

Lines changed: 2 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,10 @@ jobs:
1414
name: Generate Terminal GIFs
1515
runs-on: ubuntu-latest
1616
permissions:
17-
contents: write
18-
pull-requests: write
17+
contents: read
1918
steps:
2019
- name: Checkout PR branch
2120
uses: actions/checkout@v4
22-
with:
23-
ref: ${{ github.head_ref }}
24-
fetch-depth: 0
2521

2622
- name: Setup Rust toolchain
2723
uses: dtolnay/rust-toolchain@stable
@@ -57,7 +53,7 @@ jobs:
5753
5854
- name: Install VHS
5955
run: |
60-
VHS_VERSION="0.8.0"
56+
VHS_VERSION="0.9.0"
6157
curl -fsSL "https://github.com/charmbracelet/vhs/releases/download/v${VHS_VERSION}/vhs_${VHS_VERSION}_amd64.deb" -o vhs.deb
6258
sudo dpkg -i vhs.deb
6359
rm vhs.deb
@@ -73,83 +69,9 @@ jobs:
7369
echo "::endgroup::"
7470
done
7571
76-
- name: Commit GIFs to PR branch
77-
id: commit-gifs
78-
run: |
79-
cd actions/govbot
80-
git add tapes/*.gif --force
81-
if git diff --cached --quiet; then
82-
echo "No GIF changes to commit"
83-
echo "changed=false" >> $GITHUB_OUTPUT
84-
else
85-
git config user.name "github-actions[bot]"
86-
git config user.email "github-actions[bot]@users.noreply.github.com"
87-
git commit -m "chore: update terminal screenshot GIFs [skip ci]"
88-
git push
89-
echo "changed=true" >> $GITHUB_OUTPUT
90-
fi
91-
echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
92-
9372
- name: Upload GIFs as artifacts
9473
uses: actions/upload-artifact@v4
9574
with:
9675
name: terminal-screenshots
9776
path: actions/govbot/tapes/*.gif
9877
retention-days: 30
99-
100-
- name: Post PR comment with inline GIFs
101-
uses: actions/github-script@v7
102-
with:
103-
script: |
104-
const fs = require('fs');
105-
const path = require('path');
106-
107-
const tapesDir = 'actions/govbot/tapes';
108-
const gifs = fs.readdirSync(tapesDir).filter(f => f.endsWith('.gif')).sort();
109-
110-
if (gifs.length === 0) {
111-
console.log('No GIFs found, skipping comment');
112-
return;
113-
}
114-
115-
const sha = '${{ steps.commit-gifs.outputs.sha }}';
116-
const owner = context.repo.owner;
117-
const repo = context.repo.repo;
118-
119-
let body = `## Terminal Screenshots\n\n`;
120-
body += `> Auto-generated from \`actions/govbot/tapes/*.tape\` at ${sha.slice(0, 7)}\n\n`;
121-
122-
for (const gif of gifs) {
123-
const name = path.basename(gif, '.gif');
124-
const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${sha}/actions/govbot/tapes/${gif}`;
125-
body += `### \`${name}\`\n\n`;
126-
body += `![${name}](${rawUrl})\n\n`;
127-
}
128-
129-
// Find existing comment to update (avoids duplicate comments on re-runs)
130-
const { data: comments } = await github.rest.issues.listComments({
131-
owner,
132-
repo,
133-
issue_number: context.issue.number,
134-
});
135-
136-
const marker = '## Terminal Screenshots';
137-
const existing = comments.find(c => c.body && c.body.startsWith(marker));
138-
139-
if (existing) {
140-
await github.rest.issues.updateComment({
141-
owner,
142-
repo,
143-
comment_id: existing.id,
144-
body,
145-
});
146-
console.log(`Updated existing comment ${existing.id}`);
147-
} else {
148-
await github.rest.issues.createComment({
149-
owner,
150-
repo,
151-
issue_number: context.issue.number,
152-
body,
153-
});
154-
console.log('Created new comment');
155-
}

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ docs/action_logs/*
101101

102102
**/tmp/*
103103

104+
# VHS generated recordings (stored as CI artifacts, not in repo)
105+
*.gif
106+
104107
# Rust / Cargo
105108
target/
106109
**/target/

‎actions/govbot/src/main.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2326,6 +2326,9 @@ async fn main() -> anyhow::Result<()> {
23262326
} else {
23272327
govbot::wizard::write_default_files(&cwd)?;
23282328
}
2329+
// Exit after generating config; user runs `govbot` again
2330+
// to start the pipeline (matches the wizard's own message).
2331+
return Ok(());
23292332
}
23302333
govbot::pipeline::run_pipeline(&config_path)
23312334
}

‎actions/govbot/tapes/govbot-clone-list.gif‎

-28.3 KB
Binary file not shown.

‎actions/govbot/tapes/govbot-help.gif‎

-23.7 KB
Binary file not shown.

‎actions/govbot/tapes/logs-basic.gif‎

-360 KB
Binary file not shown.

0 commit comments

Comments
 (0)