Skip to content

Commit 9a1f01d

Browse files
GordonSmithGordonSmith
committed
feat: test against wit-bingen tests
Signed-off-by: Gordon Smith <[email protected]>
1 parent 5e1584f commit 9a1f01d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+6970
-1469
lines changed

‎.github/scripts/summarize-test-failures.sh‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#!/bin/bash
2+
# Script to summarize wit-stub-generation-test failures
3+
# Outputs a markdown summary suitable for PR comments
4+
#
5+
# Usage: summarize-test-failures.sh <test_output_file> <summary_file>
6+
# test_output_file: File containing captured test output (required)
7+
# summary_file: File to write markdown summary (default: test_summary.md)
8+
9+
set +e # Don't exit on error
10+
11+
TEST_OUTPUT_FILE="${1}"
12+
SUMMARY_FILE="${2:-test_summary.md}"
13+
14+
if [ -z "$TEST_OUTPUT_FILE" ] || [ ! -f "$TEST_OUTPUT_FILE" ]; then
15+
echo "Error: Test output file required as first argument"
16+
exit 1
17+
fi
18+
19+
echo "## wit-stub-generation-test Results" > "$SUMMARY_FILE"
20+
echo "" >> "$SUMMARY_FILE"
21+
echo "This test is expected to have failures due to known issues with tuple wrapper types." >> "$SUMMARY_FILE"
22+
echo "" >> "$SUMMARY_FILE"
23+
24+
# Check if test passed or failed
25+
TEST_EXIT_CODE=1
26+
if grep -q "100% tests passed" "$TEST_OUTPUT_FILE"; then
27+
TEST_EXIT_CODE=0
28+
fi
29+
30+
if [ $TEST_EXIT_CODE -eq 0 ]; then
31+
echo "✅ **All tests passed!**" >> "$SUMMARY_FILE"
32+
echo "" >> "$SUMMARY_FILE"
33+
echo "Great news! The wit-stub-generation-test is now fully passing." >> "$SUMMARY_FILE"
34+
else
35+
# Extract compilation statistics from ninja build output
36+
# Look for lines like "4: [6/13] Building CXX"
37+
BUILD_LINES=$(grep "Building CXX" "$TEST_OUTPUT_FILE" | head -1)
38+
39+
if [ -n "$BUILD_LINES" ]; then
40+
# Extract total from format [X/TOTAL]
41+
TOTAL_TARGETS=$(echo "$BUILD_LINES" | grep -oP '\[\d+/\K\d+' | head -1)
42+
else
43+
TOTAL_TARGETS="0"
44+
fi
45+
46+
FAILED_TARGETS=$(grep -c "^4: FAILED:" "$TEST_OUTPUT_FILE" || echo "0")
47+
PASSED_TARGETS=$((TOTAL_TARGETS - FAILED_TARGETS))
48+
49+
if [ $TOTAL_TARGETS -gt 0 ]; then
50+
PASS_PERCENT=$(awk "BEGIN {printf \"%.1f\", ($PASSED_TARGETS / $TOTAL_TARGETS) * 100}")
51+
FAIL_PERCENT=$(awk "BEGIN {printf \"%.1f\", ($FAILED_TARGETS / $TOTAL_TARGETS) * 100}")
52+
else
53+
PASS_PERCENT="0.0"
54+
FAIL_PERCENT="0.0"
55+
fi
56+
57+
# Extract error counts
58+
TOTAL_ERRORS=$(grep -c "error:" "$TEST_OUTPUT_FILE" || echo "0")
59+
60+
# Show compilation statistics
61+
echo "### Compilation Summary" >> "$SUMMARY_FILE"
62+
echo "" >> "$SUMMARY_FILE"
63+
echo "| Metric | Value |" >> "$SUMMARY_FILE"
64+
echo "|--------|-------|" >> "$SUMMARY_FILE"
65+
echo "| Total targets | $TOTAL_TARGETS |" >> "$SUMMARY_FILE"
66+
echo "| ✅ Passed | $PASSED_TARGETS ($PASS_PERCENT%) |" >> "$SUMMARY_FILE"
67+
echo "| ❌ Failed | $FAILED_TARGETS ($FAIL_PERCENT%) |" >> "$SUMMARY_FILE"
68+
echo "| Total errors | $TOTAL_ERRORS |" >> "$SUMMARY_FILE"
69+
echo "" >> "$SUMMARY_FILE"
70+
71+
# Count specific error patterns
72+
NO_MATCHING_FUNCTION=$(grep "error: no matching function for call to 'get<" "$TEST_OUTPUT_FILE" | wc -l || echo "0")
73+
RESULT_OK_ERRORS=$(grep "result_ok_wrapper" "$TEST_OUTPUT_FILE" | wc -l || echo "0")
74+
RESULT_ERR_ERRORS=$(grep "result_err_wrapper" "$TEST_OUTPUT_FILE" | wc -l || echo "0")
75+
MISSING_FILES=$(grep "fatal error:.*No such file or directory" "$TEST_OUTPUT_FILE" | wc -l || echo "0")
76+
77+
echo "### Error Breakdown" >> "$SUMMARY_FILE"
78+
echo "" >> "$SUMMARY_FILE"
79+
echo "| Error Type | Count |" >> "$SUMMARY_FILE"
80+
echo "|------------|-------|" >> "$SUMMARY_FILE"
81+
echo "| \`std::get<>\` function call errors | $NO_MATCHING_FUNCTION |" >> "$SUMMARY_FILE"
82+
echo "| \`result_ok_wrapper\` related | $RESULT_OK_ERRORS |" >> "$SUMMARY_FILE"
83+
echo "| \`result_err_wrapper\` related | $RESULT_ERR_ERRORS |" >> "$SUMMARY_FILE"
84+
echo "| Missing file errors | $MISSING_FILES |" >> "$SUMMARY_FILE"
85+
echo "" >> "$SUMMARY_FILE"
86+
87+
# Extract sample errors (first 5 unique error messages, excluding file paths)
88+
echo "### Sample Errors" >> "$SUMMARY_FILE"
89+
echo "" >> "$SUMMARY_FILE"
90+
echo '```' >> "$SUMMARY_FILE"
91+
grep "error:" "$TEST_OUTPUT_FILE" | grep -v "No such file or directory" | sed 's|/home/[^/]*/component-model-cpp/||g' | sed 's/^4: //' | sort -u | head -5 >> "$SUMMARY_FILE"
92+
echo '```' >> "$SUMMARY_FILE"
93+
echo "" >> "$SUMMARY_FILE"
94+
95+
# Check if errors increased, decreased, or stayed the same
96+
if [ -f "previous_test_summary.txt" ]; then
97+
PREV_ERRORS=$(cat previous_test_summary.txt)
98+
99+
echo "### Trend Analysis" >> "$SUMMARY_FILE"
100+
echo "" >> "$SUMMARY_FILE"
101+
102+
if [ "$TOTAL_ERRORS" -lt "$PREV_ERRORS" ]; then
103+
DIFF=$((PREV_ERRORS - TOTAL_ERRORS))
104+
echo "✅ **Improvement**: $DIFF fewer errors than previous run ($PREV_ERRORS → $TOTAL_ERRORS)" >> "$SUMMARY_FILE"
105+
elif [ "$TOTAL_ERRORS" -gt "$PREV_ERRORS" ]; then
106+
DIFF=$((TOTAL_ERRORS - PREV_ERRORS))
107+
echo "⚠️ **Regression**: $DIFF more errors than previous run ($PREV_ERRORS → $TOTAL_ERRORS)" >> "$SUMMARY_FILE"
108+
else
109+
echo "➡️ **No change**: Same number of errors as previous run ($TOTAL_ERRORS)" >> "$SUMMARY_FILE"
110+
fi
111+
echo "" >> "$SUMMARY_FILE"
112+
fi
113+
114+
# Save current error count for next run
115+
echo "$TOTAL_ERRORS" > previous_test_summary.txt
116+
fi
117+
118+
echo "### Known Issues" >> "$SUMMARY_FILE"
119+
echo "" >> "$SUMMARY_FILE"
120+
echo "The main issue is that \`result_ok_wrapper\` and \`result_err_wrapper\` types don't support \`std::get<>\` operations needed for tuple unpacking in generated code." >> "$SUMMARY_FILE"
121+
echo "" >> "$SUMMARY_FILE"
122+
echo "**Goal**: These failures should ideally decrease over time or remain stable. Any increase warrants investigation." >> "$SUMMARY_FILE"
123+
124+
echo "Summary written to $SUMMARY_FILE"
125+
cat "$SUMMARY_FILE"
126+
127+
# Always exit successfully so the workflow continues
128+
exit 0

‎.github/workflows/macos.yml‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
name: MacOS
22

3+
# Disabled: Known to fail
4+
# To re-enable, uncomment the 'on:' section below
35
on:
4-
push:
5-
branches:
6-
- trunk
7-
- main
8-
pull_request:
9-
branches:
10-
- trunk
11-
- main
12-
workflow_dispatch:
6+
workflow_dispatch: # Manual trigger only
7+
# push:
8+
# branches:
9+
# - trunk
10+
# - main
11+
# pull_request:
12+
# branches:
13+
# - trunk
14+
# - main
1315

1416
env:
1517
CTEST_OUTPUT_ON_FAILURE: 1
@@ -70,7 +72,14 @@ jobs:
7072
- name: test
7173
working-directory: build
7274
run: |
73-
ctest -VV
75+
ctest -VV -E "wit-stub-generation-test"
76+
77+
- name: test-stubs-full (allowed to fail)
78+
working-directory: build
79+
continue-on-error: true
80+
run: |
81+
echo "Running wit-stub-generation-test (failures expected and will be reported)..."
82+
ctest -VV -R "wit-stub-generation-test"
7483
7584
- name: Upload error logs
7685
if: ${{ failure() || cancelled() }}

‎.github/workflows/release.yml‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -42,23 +42,25 @@ jobs:
4242
build/cmcpp-*.exe
4343
extra-deps: ""
4444

45-
- os: macos-14
46-
preset: linux-ninja-Release
47-
build-preset: linux-ninja-Release
48-
cpack-generators: "TGZ;ZIP"
49-
package-patterns: |
50-
build/cmcpp-*.tar.gz
51-
build/cmcpp-*.zip
52-
extra-deps: |
53-
brew install \
54-
pkg-config \
55-
autoconf \
56-
autoconf-archive \
57-
automake \
58-
coreutils \
59-
libtool \
60-
cmake \
61-
ninja
45+
# Disabled: Known to fail
46+
# To re-enable, uncomment the macOS section below
47+
# - os: macos-14
48+
# preset: linux-ninja-Release
49+
# build-preset: linux-ninja-Release
50+
# cpack-generators: "TGZ;ZIP"
51+
# package-patterns: |
52+
# build/cmcpp-*.tar.gz
53+
# build/cmcpp-*.zip
54+
# extra-deps: |
55+
# brew install \
56+
# pkg-config \
57+
# autoconf \
58+
# autoconf-archive \
59+
# automake \
60+
# coreutils \
61+
# libtool \
62+
# cmake \
63+
# ninja
6264

6365
runs-on: ${{ matrix.os }}
6466
permissions:
@@ -117,7 +119,13 @@ jobs:
117119
- name: Run Tests
118120
working-directory: build
119121
run: |
120-
ctest ${{ matrix.cpack-config || '' }} -VV
122+
ctest ${{ matrix.cpack-config || '' }} -VV -E "wit-stub-generation-test"
123+
124+
- name: Upload test summary
125+
uses: actions/upload-artifact@v4
126+
with:
127+
name: ${{ matrix.os }}-test-summary
128+
path: build/test_summary.md
121129

122130
- name: Create Packages
123131
working-directory: build

‎.github/workflows/ubuntu.yml‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,57 @@ jobs:
7777
- name: test
7878
working-directory: build
7979
run: |
80-
ctest -VV
80+
ctest -VV -E "wit-stub-generation-test"
81+
82+
- name: test-stubs-full (allowed to fail)
83+
working-directory: build
84+
continue-on-error: true
85+
run: |
86+
echo "Running wit-stub-generation-test (failures expected and will be reported)..."
87+
ctest -VV -R "wit-stub-generation-test" > test_output.txt 2>&1 || true
88+
cat test_output.txt
89+
../.github/scripts/summarize-test-failures.sh test_output.txt test_summary.md
90+
91+
- name: Comment PR with test summary
92+
if: github.event_name == 'pull_request'
93+
uses: actions/github-script@v7
94+
with:
95+
github-token: ${{ secrets.GITHUB_TOKEN }}
96+
script: |
97+
const fs = require('fs');
98+
const summary = fs.readFileSync('build/test_summary.md', 'utf8');
99+
100+
// Find existing comment
101+
const { data: comments } = await github.rest.issues.listComments({
102+
owner: context.repo.owner,
103+
repo: context.repo.repo,
104+
issue_number: context.issue.number,
105+
});
106+
107+
const botComment = comments.find(comment =>
108+
comment.user.type === 'Bot' &&
109+
comment.body.includes('wit-stub-generation-test Results')
110+
);
111+
112+
const commentBody = `### Ubuntu Test Results\n\n${summary}\n\n---\n*Updated: ${new Date().toISOString()}*`;
113+
114+
if (botComment) {
115+
// Update existing comment
116+
await github.rest.issues.updateComment({
117+
owner: context.repo.owner,
118+
repo: context.repo.repo,
119+
comment_id: botComment.id,
120+
body: commentBody
121+
});
122+
} else {
123+
// Create new comment
124+
await github.rest.issues.createComment({
125+
owner: context.repo.owner,
126+
repo: context.repo.repo,
127+
issue_number: context.issue.number,
128+
body: commentBody
129+
});
130+
}
81131
82132
- name: Upload coverage reports to Codecov
83133
uses: codecov/[email protected]

‎.github/workflows/windows.yml‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,58 @@ jobs:
7373
- name: test
7474
working-directory: build
7575
run: |
76-
ctest -C Debug -VV
76+
ctest -C Debug -VV -E "wit-stub-generation-test"
77+
78+
- name: test-stubs-full (allowed to fail)
79+
working-directory: build
80+
continue-on-error: true
81+
shell: bash
82+
run: |
83+
echo "Running wit-stub-generation-test (failures expected and will be reported)..."
84+
ctest -C Debug -VV -R "wit-stub-generation-test" > test_output.txt 2>&1 || true
85+
cat test_output.txt
86+
../.github/scripts/summarize-test-failures.sh test_output.txt test_summary.md
87+
88+
- name: Comment PR with test summary
89+
if: github.event_name == 'pull_request'
90+
uses: actions/github-script@v7
91+
with:
92+
github-token: ${{ secrets.GITHUB_TOKEN }}
93+
script: |
94+
const fs = require('fs');
95+
const summary = fs.readFileSync('build/test_summary.md', 'utf8');
96+
97+
// Find existing comment
98+
const { data: comments } = await github.rest.issues.listComments({
99+
owner: context.repo.owner,
100+
repo: context.repo.repo,
101+
issue_number: context.issue.number,
102+
});
103+
104+
const botComment = comments.find(comment =>
105+
comment.user.type === 'Bot' &&
106+
comment.body.includes('wit-stub-generation-test Results')
107+
);
108+
109+
const commentBody = `### Windows Test Results\n\n${summary}\n\n---\n*Updated: ${new Date().toISOString()}*`;
110+
111+
if (botComment) {
112+
// Update existing comment
113+
await github.rest.issues.updateComment({
114+
owner: context.repo.owner,
115+
repo: context.repo.repo,
116+
comment_id: botComment.id,
117+
body: commentBody
118+
});
119+
} else {
120+
// Create new comment
121+
await github.rest.issues.createComment({
122+
owner: context.repo.owner,
123+
repo: context.repo.repo,
124+
issue_number: context.issue.number,
125+
body: commentBody
126+
});
127+
}
77128
78129
- name: Upload error logs
79130
if: ${{ failure() || cancelled() }}

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,7 @@ antlr-*.jar
1616
# Python cache files
1717
__pycache__/
1818
*.pyc
19+
20+
# Generated test stubs
21+
/test/generated_stubs/
22+
/test/test_stubs_sample/

‎include/cmcpp.hpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,16 @@
22
#define CMCPP_HPP
33

44
#include <cmcpp/context.hpp>
5+
#include <cmcpp/monostate.hpp>
6+
#include <cmcpp/bool.hpp>
57
#include <cmcpp/integer.hpp>
68
#include <cmcpp/float.hpp>
79
#include <cmcpp/string.hpp>
810
#include <cmcpp/error_context.hpp>
911
#include <cmcpp/flags.hpp>
10-
#include <cmcpp/list.hpp>
1112
#include <cmcpp/tuple.hpp>
13+
#include <cmcpp/record.hpp>
14+
#include <cmcpp/list.hpp>
1215
#include <cmcpp/variant.hpp>
1316
#include <cmcpp/func.hpp>
1417
#include <cmcpp/lower.hpp>

0 commit comments

Comments
 (0)