|
| 1 | +#!/bin/bash |
| 2 | +set -e # Exit on error |
| 3 | + |
| 4 | +# Script to run all test configurations |
| 5 | +# Usage: ./run-tests.sh <config_file> |
| 6 | + |
| 7 | +CONFIG_FILE="${1:-.github/workflows/test-matrix.json}" |
| 8 | + |
| 9 | +# Initialize counters and tracking |
| 10 | +total=0 |
| 11 | +passed=0 |
| 12 | +failed=0 |
| 13 | +declare -a failed_configs |
| 14 | +declare -a failed_log_dirs |
| 15 | + |
| 16 | +# Initialize GitHub Summary if running in GitHub Actions |
| 17 | +if [ -n "$GITHUB_STEP_SUMMARY" ]; then |
| 18 | + echo "# Build and Execution Test Results" >> "$GITHUB_STEP_SUMMARY" |
| 19 | + echo "" >> "$GITHUB_STEP_SUMMARY" |
| 20 | + echo "| # | Configuration | Status | Details |" >> "$GITHUB_STEP_SUMMARY" |
| 21 | + echo "|---|---------------|--------|---------|" >> "$GITHUB_STEP_SUMMARY" |
| 22 | +fi |
| 23 | + |
| 24 | +# Function to add result to summary |
| 25 | +add_to_summary() { |
| 26 | + local num=$1 |
| 27 | + local config=$2 |
| 28 | + local status=$3 |
| 29 | + local details=$4 |
| 30 | + |
| 31 | + if [ -n "$GITHUB_STEP_SUMMARY" ]; then |
| 32 | + echo "| $num | $config | $status | $details |" >> "$GITHUB_STEP_SUMMARY" |
| 33 | + fi |
| 34 | +} |
| 35 | + |
| 36 | +# Function to build, execute and verify a configuration |
| 37 | +test_configuration() { |
| 38 | + local compiler=$1 |
| 39 | + local ext=$2 |
| 40 | + local target=$3 |
| 41 | + local model=$4 |
| 42 | + local uart=$5 |
| 43 | + local build=$6 |
| 44 | + |
| 45 | + local config_name="$build+$target with $compiler" |
| 46 | + local failure_reason="" |
| 47 | + local log_file="./logs/${target}_${compiler}_${build}.log" |
| 48 | + |
| 49 | + # Create logs directory |
| 50 | + mkdir -p "./logs" |
| 51 | + |
| 52 | + # Initialize log file with header |
| 53 | + echo "=========================================" > "$log_file" |
| 54 | + echo "Configuration: $config_name" >> "$log_file" |
| 55 | + echo "Date: $(date)" >> "$log_file" |
| 56 | + echo "=========================================" >> "$log_file" |
| 57 | + echo "" >> "$log_file" |
| 58 | + |
| 59 | + echo "" |
| 60 | + echo "==========================================" |
| 61 | + echo "[$total/60] Testing: $config_name" |
| 62 | + echo "==========================================" |
| 63 | + |
| 64 | + # Build |
| 65 | + echo "▶ Building..." |
| 66 | + echo "=========================================" >> "$log_file" |
| 67 | + echo "BUILD OUTPUT" >> "$log_file" |
| 68 | + echo "=========================================" >> "$log_file" |
| 69 | + if ! cbuild Hello.csolution.yml --packs \ |
| 70 | + --context "Hello.$build+$target" \ |
| 71 | + --toolchain "$compiler" --rebuild 2>&1 | tee -a "$log_file"; then |
| 72 | + failure_reason="Build failed" |
| 73 | + echo "✗ $failure_reason" |
| 74 | + echo "" >> "$log_file" |
| 75 | + echo "Result: FAILED - $failure_reason" >> "$log_file" |
| 76 | + failed=$((failed + 1)) |
| 77 | + failed_configs+=("[$total] $config_name - $failure_reason") |
| 78 | + failed_log_dirs+=("$log_file") |
| 79 | + add_to_summary "$total" "$config_name" "❌ Failed" "$failure_reason" |
| 80 | + return 1 |
| 81 | + fi |
| 82 | + echo "✅ Build successful" |
| 83 | + |
| 84 | + # Execute |
| 85 | + echo "▶ Executing on $model..." |
| 86 | + echo "" >> "$log_file" |
| 87 | + echo "=========================================" >> "$log_file" |
| 88 | + echo "EXECUTION OUTPUT" >> "$log_file" |
| 89 | + echo "=========================================" >> "$log_file" |
| 90 | + if ! "$model" \ |
| 91 | + -a "./out/Hello/$target/$build/$compiler/Hello.$ext" \ |
| 92 | + -f "./FVP/$model/fvp_config.txt" \ |
| 93 | + -C "$uart.out_file=./out/Hello/$target/$build/$compiler/fvp_stdout.log" \ |
| 94 | + --simlimit 60 --stat 2>&1 | tee -a "$log_file"; then |
| 95 | + failure_reason="Execution failed" |
| 96 | + echo "❌ $failure_reason" |
| 97 | + echo "" >> "$log_file" |
| 98 | + echo "Result: FAILED - $failure_reason" >> "$log_file" |
| 99 | + failed=$((failed + 1)) |
| 100 | + failed_configs+=("[$total] $config_name - $failure_reason") |
| 101 | + failed_log_dirs+=("$log_file") |
| 102 | + add_to_summary "$total" "$config_name" "❌ Failed" "$failure_reason" |
| 103 | + return 1 |
| 104 | + fi |
| 105 | + echo "✅ Execution successful" |
| 106 | + |
| 107 | + echo "Actual UART output:" |
| 108 | + cat "./out/Hello/$target/$build/$compiler/fvp_stdout.log" || echo "Could not read log file" |
| 109 | + |
| 110 | + if grep -q "Hello World 100" "./out/Hello/$target/$build/$compiler/fvp_stdout.log"; then |
| 111 | + echo "✅ Test PASSED" |
| 112 | + passed=$((passed + 1)) |
| 113 | + add_to_summary "$total" "$config_name" "✅ Passed" "" |
| 114 | + # Clean up log for passed test |
| 115 | + rm -f "$log_file" |
| 116 | + return 0 |
| 117 | + else |
| 118 | + failure_reason="Verification failed - 'Hello World 100' not found" |
| 119 | + echo "❌ $failure_reason" |
| 120 | + |
| 121 | + # Add UART output to log file |
| 122 | + echo "" >> "$log_file" |
| 123 | + echo "=========================================" >> "$log_file" |
| 124 | + echo "UART OUTPUT" >> "$log_file" |
| 125 | + echo "=========================================" >> "$log_file" |
| 126 | + cat "./out/Hello/$target/$build/$compiler/fvp_stdout.log" >> "$log_file" 2>/dev/null || echo "Could not read UART log" >> "$log_file" |
| 127 | + echo "" >> "$log_file" |
| 128 | + echo "Result: FAILED - $failure_reason" >> "$log_file" |
| 129 | + |
| 130 | + failed=$((failed + 1)) |
| 131 | + failed_configs+=("[$total] $config_name - $failure_reason") |
| 132 | + failed_log_dirs+=("$log_file") |
| 133 | + add_to_summary "$total" "$config_name" "❌ Failed" "$failure_reason" |
| 134 | + return 1 |
| 135 | + fi |
| 136 | +} |
| 137 | + |
| 138 | +# Main execution loop |
| 139 | +echo "Reading configurations from: $CONFIG_FILE" |
| 140 | + |
| 141 | +# Generate all combinations from compilers × targets × builds |
| 142 | +while IFS= read -r compiler_data; do |
| 143 | + compiler=$(echo "$compiler_data" | jq -r '.name') |
| 144 | + ext=$(echo "$compiler_data" | jq -r '.ext') |
| 145 | + |
| 146 | + while IFS= read -r target_data; do |
| 147 | + target=$(echo "$target_data" | jq -r '.type') |
| 148 | + model=$(echo "$target_data" | jq -r '.model') |
| 149 | + uart=$(echo "$target_data" | jq -r '.uart') |
| 150 | + |
| 151 | + while IFS= read -r build; do |
| 152 | + total=$((total + 1)) |
| 153 | + test_configuration "$compiler" "$ext" "$target" "$model" "$uart" "$build" || true |
| 154 | + done < <(jq -r '.builds[]' "$CONFIG_FILE") |
| 155 | + |
| 156 | + done < <(jq -c '.targets[]' "$CONFIG_FILE") |
| 157 | + |
| 158 | +done < <(jq -c '.compilers[]' "$CONFIG_FILE") |
| 159 | + |
| 160 | +# Add summary statistics to GitHub Summary |
| 161 | +if [ -n "$GITHUB_STEP_SUMMARY" ]; then |
| 162 | + echo "" >> "$GITHUB_STEP_SUMMARY" |
| 163 | + echo "## Summary" >> "$GITHUB_STEP_SUMMARY" |
| 164 | + echo "" >> "$GITHUB_STEP_SUMMARY" |
| 165 | + echo "- **Total Configurations:** $total" >> "$GITHUB_STEP_SUMMARY" |
| 166 | + echo "- **✅ Passed:** $passed" >> "$GITHUB_STEP_SUMMARY" |
| 167 | + echo "- **❌ Failed:** $failed" >> "$GITHUB_STEP_SUMMARY" |
| 168 | +fi |
| 169 | + |
| 170 | +# Print comprehensive summary |
| 171 | +echo "" |
| 172 | +echo "==========================================" |
| 173 | +echo " FINAL SUMMARY" |
| 174 | +echo "==========================================" |
| 175 | +echo "Total configurations: $total" |
| 176 | +echo "Passed: $passed" |
| 177 | +echo "Failed: $failed" |
| 178 | +echo "" |
| 179 | + |
| 180 | +if [ $failed -gt 0 ]; then |
| 181 | + echo "❌ FAILED CONFIGURATIONS:" |
| 182 | + echo "==========================================" |
| 183 | + for fail_info in "${failed_configs[@]}"; do |
| 184 | + echo "$fail_info" |
| 185 | + done |
| 186 | + echo "==========================================" |
| 187 | + echo "" |
| 188 | + echo "Logs for failed configurations saved in: ./logs/" |
| 189 | + echo "Total failed log files: ${#failed_log_dirs[@]}" |
| 190 | + exit 1 |
| 191 | +else |
| 192 | + echo "✅ All tests passed!" |
| 193 | + # Clean up logs directory if all tests passed |
| 194 | + rm -rf ./logs |
| 195 | +fi |
0 commit comments