Skip to content

Commit 968dbae

Browse files
GordonSmithGordonSmith
committed
feat: test against wit-bingen tests
Signed-off-by: Gordon Smith <[email protected]>
1 parent 5e1584f commit 968dbae

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+6943
-1469
lines changed

‎.github/scripts/summarize-test-failures.sh‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#!/bin/bash
2+
# Script to summarize wit-stub-generation-test failures
3+
# Outputs a markdown summary suitable for PR comments
4+
#
5+
# Usage: summarize-test-failures.sh <test_output_file> <summary_file>
6+
# test_output_file: File containing captured test output (required)
7+
# summary_file: File to write markdown summary (default: test_summary.md)
8+
9+
set +e # Don't exit on error
10+
11+
TEST_OUTPUT_FILE="${1}"
12+
SUMMARY_FILE="${2:-test_summary.md}"
13+
14+
if [ -z "$TEST_OUTPUT_FILE" ] || [ ! -f "$TEST_OUTPUT_FILE" ]; then
15+
echo "Error: Test output file required as first argument"
16+
exit 1
17+
fi
18+
19+
echo "## wit-stub-generation-test Results" > "$SUMMARY_FILE"
20+
echo "" >> "$SUMMARY_FILE"
21+
echo "This test is expected to have failures due to known issues with tuple wrapper types." >> "$SUMMARY_FILE"
22+
echo "" >> "$SUMMARY_FILE"
23+
24+
# Check if test passed or failed
25+
TEST_EXIT_CODE=1
26+
if grep -q "100% tests passed" "$TEST_OUTPUT_FILE"; then
27+
TEST_EXIT_CODE=0
28+
fi
29+
30+
if [ $TEST_EXIT_CODE -eq 0 ]; then
31+
echo "✅ **All tests passed!**" >> "$SUMMARY_FILE"
32+
echo "" >> "$SUMMARY_FILE"
33+
echo "Great news! The wit-stub-generation-test is now fully passing." >> "$SUMMARY_FILE"
34+
else
35+
# Extract compilation statistics from ninja build output
36+
# Look for lines like "4: [6/13] Building CXX"
37+
BUILD_LINES=$(grep "Building CXX" "$TEST_OUTPUT_FILE" | head -1)
38+
39+
if [ -n "$BUILD_LINES" ]; then
40+
# Extract total from format [X/TOTAL]
41+
TOTAL_TARGETS=$(echo "$BUILD_LINES" | grep -oP '\[\d+/\K\d+' | head -1)
42+
else
43+
TOTAL_TARGETS="0"
44+
fi
45+
46+
FAILED_TARGETS=$(grep -c "^4: FAILED:" "$TEST_OUTPUT_FILE" || echo "0")
47+
PASSED_TARGETS=$((TOTAL_TARGETS - FAILED_TARGETS))
48+
49+
if [ $TOTAL_TARGETS -gt 0 ]; then
50+
PASS_PERCENT=$(awk "BEGIN {printf \"%.1f\", ($PASSED_TARGETS / $TOTAL_TARGETS) * 100}")
51+
FAIL_PERCENT=$(awk "BEGIN {printf \"%.1f\", ($FAILED_TARGETS / $TOTAL_TARGETS) * 100}")
52+
else
53+
PASS_PERCENT="0.0"
54+
FAIL_PERCENT="0.0"
55+
fi
56+
57+
# Extract error counts
58+
TOTAL_ERRORS=$(grep -c "error:" "$TEST_OUTPUT_FILE" || echo "0")
59+
60+
# Show compilation statistics
61+
echo "### Compilation Summary" >> "$SUMMARY_FILE"
62+
echo "" >> "$SUMMARY_FILE"
63+
echo "| Metric | Value |" >> "$SUMMARY_FILE"
64+
echo "|--------|-------|" >> "$SUMMARY_FILE"
65+
echo "| Total targets | $TOTAL_TARGETS |" >> "$SUMMARY_FILE"
66+
echo "| ✅ Passed | $PASSED_TARGETS ($PASS_PERCENT%) |" >> "$SUMMARY_FILE"
67+
echo "| ❌ Failed | $FAILED_TARGETS ($FAIL_PERCENT%) |" >> "$SUMMARY_FILE"
68+
echo "| Total errors | $TOTAL_ERRORS |" >> "$SUMMARY_FILE"
69+
echo "" >> "$SUMMARY_FILE"
70+
71+
# Count specific error patterns
72+
NO_MATCHING_FUNCTION=$(grep "error: no matching function for call to 'get<" "$TEST_OUTPUT_FILE" | wc -l || echo "0")
73+
RESULT_OK_ERRORS=$(grep "result_ok_wrapper" "$TEST_OUTPUT_FILE" | wc -l || echo "0")
74+
RESULT_ERR_ERRORS=$(grep "result_err_wrapper" "$TEST_OUTPUT_FILE" | wc -l || echo "0")
75+
MISSING_FILES=$(grep "fatal error:.*No such file or directory" "$TEST_OUTPUT_FILE" | wc -l || echo "0")
76+
77+
echo "### Error Breakdown" >> "$SUMMARY_FILE"
78+
echo "" >> "$SUMMARY_FILE"
79+
echo "| Error Type | Count |" >> "$SUMMARY_FILE"
80+
echo "|------------|-------|" >> "$SUMMARY_FILE"
81+
echo "| \`std::get<>\` function call errors | $NO_MATCHING_FUNCTION |" >> "$SUMMARY_FILE"
82+
echo "| \`result_ok_wrapper\` related | $RESULT_OK_ERRORS |" >> "$SUMMARY_FILE"
83+
echo "| \`result_err_wrapper\` related | $RESULT_ERR_ERRORS |" >> "$SUMMARY_FILE"
84+
echo "| Missing file errors | $MISSING_FILES |" >> "$SUMMARY_FILE"
85+
echo "" >> "$SUMMARY_FILE"
86+
87+
# Extract sample errors (first 5 unique error messages, excluding file paths)
88+
echo "### Sample Errors" >> "$SUMMARY_FILE"
89+
echo "" >> "$SUMMARY_FILE"
90+
echo '```' >> "$SUMMARY_FILE"
91+
grep "error:" "$TEST_OUTPUT_FILE" | grep -v "No such file or directory" | sed 's|/home/[^/]*/component-model-cpp/||g' | sed 's/^4: //' | sort -u | head -5 >> "$SUMMARY_FILE"
92+
echo '```' >> "$SUMMARY_FILE"
93+
echo "" >> "$SUMMARY_FILE"
94+
95+
# Check if errors increased, decreased, or stayed the same
96+
if [ -f "previous_test_summary.txt" ]; then
97+
PREV_ERRORS=$(cat previous_test_summary.txt)
98+
99+
echo "### Trend Analysis" >> "$SUMMARY_FILE"
100+
echo "" >> "$SUMMARY_FILE"
101+
102+
if [ "$TOTAL_ERRORS" -lt "$PREV_ERRORS" ]; then
103+
DIFF=$((PREV_ERRORS - TOTAL_ERRORS))
104+
echo "✅ **Improvement**: $DIFF fewer errors than previous run ($PREV_ERRORS → $TOTAL_ERRORS)" >> "$SUMMARY_FILE"
105+
elif [ "$TOTAL_ERRORS" -gt "$PREV_ERRORS" ]; then
106+
DIFF=$((TOTAL_ERRORS - PREV_ERRORS))
107+
echo "⚠️ **Regression**: $DIFF more errors than previous run ($PREV_ERRORS → $TOTAL_ERRORS)" >> "$SUMMARY_FILE"
108+
else
109+
echo "➡️ **No change**: Same number of errors as previous run ($TOTAL_ERRORS)" >> "$SUMMARY_FILE"
110+
fi
111+
echo "" >> "$SUMMARY_FILE"
112+
fi
113+
114+
# Save current error count for next run
115+
echo "$TOTAL_ERRORS" > previous_test_summary.txt
116+
fi
117+
118+
echo "### Known Issues" >> "$SUMMARY_FILE"
119+
echo "" >> "$SUMMARY_FILE"
120+
echo "The main issue is that \`result_ok_wrapper\` and \`result_err_wrapper\` types don't support \`std::get<>\` operations needed for tuple unpacking in generated code." >> "$SUMMARY_FILE"
121+
echo "" >> "$SUMMARY_FILE"
122+
echo "**Goal**: These failures should ideally decrease over time or remain stable. Any increase warrants investigation." >> "$SUMMARY_FILE"
123+
124+
echo "Summary written to $SUMMARY_FILE"
125+
cat "$SUMMARY_FILE"
126+
127+
# Always exit successfully so the workflow continues
128+
exit 0

‎.github/workflows/macos.yml‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
name: MacOS
22

3+
# Disabled: Known to fail
4+
# To re-enable, uncomment the 'on:' section below
35
on:
4-
push:
5-
branches:
6-
- trunk
7-
- main
8-
pull_request:
9-
branches:
10-
- trunk
11-
- main
12-
workflow_dispatch:
6+
workflow_dispatch: # Manual trigger only
7+
# push:
8+
# branches:
9+
# - trunk
10+
# - main
11+
# pull_request:
12+
# branches:
13+
# - trunk
14+
# - main
1315

1416
env:
1517
CTEST_OUTPUT_ON_FAILURE: 1
@@ -70,7 +72,14 @@ jobs:
7072
- name: test
7173
working-directory: build
7274
run: |
73-
ctest -VV
75+
ctest -VV -E "wit-stub-generation-test"
76+
77+
- name: test-stubs-full (allowed to fail)
78+
working-directory: build
79+
continue-on-error: true
80+
run: |
81+
echo "Running wit-stub-generation-test (failures expected and will be reported)..."
82+
ctest -VV -R "wit-stub-generation-test"
7483
7584
- name: Upload error logs
7685
if: ${{ failure() || cancelled() }}

‎.github/workflows/release.yml‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -42,23 +42,25 @@ jobs:
4242
build/cmcpp-*.exe
4343
extra-deps: ""
4444

45-
- os: macos-14
46-
preset: linux-ninja-Release
47-
build-preset: linux-ninja-Release
48-
cpack-generators: "TGZ;ZIP"
49-
package-patterns: |
50-
build/cmcpp-*.tar.gz
51-
build/cmcpp-*.zip
52-
extra-deps: |
53-
brew install \
54-
pkg-config \
55-
autoconf \
56-
autoconf-archive \
57-
automake \
58-
coreutils \
59-
libtool \
60-
cmake \
61-
ninja
45+
# Disabled: Known to fail
46+
# To re-enable, uncomment the macOS section below
47+
# - os: macos-14
48+
# preset: linux-ninja-Release
49+
# build-preset: linux-ninja-Release
50+
# cpack-generators: "TGZ;ZIP"
51+
# package-patterns: |
52+
# build/cmcpp-*.tar.gz
53+
# build/cmcpp-*.zip
54+
# extra-deps: |
55+
# brew install \
56+
# pkg-config \
57+
# autoconf \
58+
# autoconf-archive \
59+
# automake \
60+
# coreutils \
61+
# libtool \
62+
# cmake \
63+
# ninja
6264

6365
runs-on: ${{ matrix.os }}
6466
permissions:
@@ -117,7 +119,13 @@ jobs:
117119
- name: Run Tests
118120
working-directory: build
119121
run: |
120-
ctest ${{ matrix.cpack-config || '' }} -VV
122+
ctest ${{ matrix.cpack-config || '' }} -VV -E "wit-stub-generation-test"
123+
124+
- name: Upload test summary
125+
uses: actions/upload-artifact@v4
126+
with:
127+
name: ${{ matrix.os }}-test-summary
128+
path: build/test_summary.md
121129

122130
- name: Create Packages
123131
working-directory: build

‎.github/workflows/ubuntu.yml‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ env:
1515
CTEST_OUTPUT_ON_FAILURE: 1
1616
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
1717

18+
permissions:
19+
contents: read
20+
pull-requests: write
21+
issues: write
22+
1823
jobs:
1924
build:
2025
strategy:
@@ -77,7 +82,66 @@ jobs:
7782
- name: test
7883
working-directory: build
7984
run: |
80-
ctest -VV
85+
ctest -VV -E "wit-stub-generation-test"
86+
87+
- name: test-stubs-full (allowed to fail)
88+
working-directory: build
89+
continue-on-error: true
90+
run: |
91+
echo "Running wit-stub-generation-test (failures expected and will be reported)..."
92+
ctest -VV -R "wit-stub-generation-test" > test_output.txt 2>&1 || true
93+
cat test_output.txt
94+
../.github/scripts/summarize-test-failures.sh test_output.txt test_summary.md
95+
96+
- name: Comment PR with test summary
97+
if: github.event_name == 'pull_request'
98+
uses: actions/github-script@v7
99+
with:
100+
github-token: ${{ secrets.GITHUB_TOKEN }}
101+
script: |
102+
const fs = require('fs');
103+
const path = require('path');
104+
105+
// Check if summary file exists
106+
const summaryPath = path.join('build', 'test_summary.md');
107+
if (!fs.existsSync(summaryPath)) {
108+
console.log('Test summary file not found, skipping PR comment');
109+
return;
110+
}
111+
112+
const summary = fs.readFileSync(summaryPath, 'utf8');
113+
114+
// Find existing comment
115+
const { data: comments } = await github.rest.issues.listComments({
116+
owner: context.repo.owner,
117+
repo: context.repo.repo,
118+
issue_number: context.issue.number,
119+
});
120+
121+
const botComment = comments.find(comment =>
122+
comment.user.type === 'Bot' &&
123+
comment.body.includes('wit-stub-generation-test Results')
124+
);
125+
126+
const commentBody = `### Ubuntu Test Results\n\n${summary}\n\n---\n*Updated: ${new Date().toISOString()}*`;
127+
128+
if (botComment) {
129+
// Update existing comment
130+
await github.rest.issues.updateComment({
131+
owner: context.repo.owner,
132+
repo: context.repo.repo,
133+
comment_id: botComment.id,
134+
body: commentBody
135+
});
136+
} else {
137+
// Create new comment
138+
await github.rest.issues.createComment({
139+
owner: context.repo.owner,
140+
repo: context.repo.repo,
141+
issue_number: context.issue.number,
142+
body: commentBody
143+
});
144+
}
81145
82146
- name: Upload coverage reports to Codecov
83147
uses: codecov/[email protected]

‎.github/workflows/windows.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,17 @@ jobs:
7373
- name: test
7474
working-directory: build
7575
run: |
76-
ctest -C Debug -VV
76+
ctest -C Debug -VV -E "wit-stub-generation-test"
77+
78+
- name: test-stubs-full (allowed to fail)
79+
working-directory: build
80+
continue-on-error: true
81+
shell: bash
82+
run: |
83+
echo "Running wit-stub-generation-test (failures expected and will be reported)..."
84+
ctest -C Debug -VV -R "wit-stub-generation-test" > test_output.txt 2>&1 || true
85+
cat test_output.txt
86+
../.github/scripts/summarize-test-failures.sh test_output.txt test_summary.md
7787
7888
- name: Upload error logs
7989
if: ${{ failure() || cancelled() }}

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,7 @@ antlr-*.jar
1616
# Python cache files
1717
__pycache__/
1818
*.pyc
19+
20+
# Generated test stubs
21+
/test/generated_stubs/
22+
/test/test_stubs_sample/

‎include/cmcpp.hpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,16 @@
22
#define CMCPP_HPP
33

44
#include <cmcpp/context.hpp>
5+
#include <cmcpp/monostate.hpp>
6+
#include <cmcpp/bool.hpp>
57
#include <cmcpp/integer.hpp>
68
#include <cmcpp/float.hpp>
79
#include <cmcpp/string.hpp>
810
#include <cmcpp/error_context.hpp>
911
#include <cmcpp/flags.hpp>
10-
#include <cmcpp/list.hpp>
1112
#include <cmcpp/tuple.hpp>
13+
#include <cmcpp/record.hpp>
14+
#include <cmcpp/list.hpp>
1215
#include <cmcpp/variant.hpp>
1316
#include <cmcpp/func.hpp>
1417
#include <cmcpp/lower.hpp>

‎include/cmcpp/bool.hpp‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#ifndef CMCPP_BOOL_HPP
2+
#define CMCPP_BOOL_HPP
3+
4+
#include "context.hpp"
5+
#include "integer.hpp"
6+
#include "util.hpp"
7+
8+
namespace cmcpp
9+
{
10+
// Boolean ------------------------------------------------------------------
11+
template <Boolean T>
12+
inline void store(LiftLowerContext &cx, const T &v, uint32_t ptr)
13+
{
14+
uint8_t byte = v ? 1 : 0;
15+
integer::store<uint8_t>(cx, byte, ptr);
16+
}
17+
18+
template <Boolean T>
19+
inline WasmValVector lower_flat(LiftLowerContext &cx, const T &v)
20+
{
21+
using WasmValType = WasmValTypeTrait<ValTrait<T>::flat_types[0]>::type;
22+
return {static_cast<WasmValType>(v)};
23+
}
24+
25+
template <Boolean T>
26+
inline T load(const LiftLowerContext &cx, uint32_t ptr)
27+
{
28+
return convert_int_to_bool(integer::load<uint8_t>(cx, ptr));
29+
}
30+
31+
template <Boolean T>
32+
inline T lift_flat(const LiftLowerContext &cx, const CoreValueIter &vi)
33+
{
34+
return convert_int_to_bool(vi.next<int32_t>());
35+
}
36+
}
37+
38+
#endif // CMCPP_BOOL_HPP

0 commit comments

Comments
 (0)