Initial assessment and plan for implementing FML Execution Validation… #88
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: QA Report | ||
| on: | ||
| pull_request: | ||
| branches: [ main, develop ] | ||
| push: | ||
| branches: [ main, develop ] | ||
| jobs: | ||
| qa-report: | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| matrix: | ||
| node-version: [16, 18, 20] | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Setup Node.js ${{ matrix.node-version }} | ||
| uses: actions/setup-node@v4 | ||
| with: | ||
| node-version: ${{ matrix.node-version }} | ||
| cache: 'npm' | ||
| - name: Install dependencies | ||
| run: npm ci | ||
| - name: Run linting | ||
| id: lint | ||
| continue-on-error: true | ||
| run: | | ||
| npm run lint 2>&1 | tee lint-output.txt | ||
| - name: Build project | ||
| id: build | ||
| continue-on-error: true | ||
| run: | | ||
| npm run build 2>&1 | tee build-output.txt | ||
| - name: Run tests with coverage | ||
| id: coverage | ||
| continue-on-error: true | ||
| run: | | ||
| ./node_modules/.bin/jest --testMatch="**/tests/**/*.test.ts" --coverage --coverageReporters=text-lcov 2>&1 | tee coverage-output.txt | ||
| - name: FML Compilation Tests | ||
| id: fml-compilation | ||
| continue-on-error: true | ||
| run: | | ||
| echo "=== FML Compilation Test Results ===" | ||
| ./node_modules/.bin/jest --testMatch="**/tests/**/*.test.ts" --testPathPattern="fml-compiler|fhir-mapping-language" --verbose 2>&1 | tee fml-compilation-output.txt | ||
| - name: FML Execution Tests | ||
| id: fml-execution | ||
| continue-on-error: true | ||
| run: | | ||
| echo "=== FML Execution Test Results ===" | ||
| ./node_modules/.bin/jest --testMatch="**/tests/**/*.test.ts" --testPathPattern="structure-map-executor|fhirpath-integration" --verbose 2>&1 | tee fml-execution-output.txt | ||
| - name: FHIR API Tests | ||
| id: fhir-api | ||
| continue-on-error: true | ||
| run: | | ||
| echo "=== FHIR API Test Results ===" | ||
| ./node_modules/.bin/jest --testMatch="**/tests/**/*.test.ts" --testPathPattern="api|enhanced-api" --verbose 2>&1 | tee fhir-api-output.txt | ||
| - name: Validation & Core Tests | ||
| id: validation-core | ||
| continue-on-error: true | ||
| run: | | ||
| echo "=== Validation & Core Test Results ===" | ||
| ./node_modules/.bin/jest --testMatch="**/tests/**/*.test.ts" --testPathPattern="validation-service|fml-runner|structure-map-retriever|enhanced-tokenizer" --verbose 2>&1 | tee validation-core-output.txt | ||
| - name: Generate QA Summary Table and Post PR Comment | ||
| if: always() | ||
| uses: actions/github-script@v7 | ||
| with: | ||
| script: | | ||
| const fs = require('fs'); | ||
| // Helper function to read file safely | ||
| function readOutputFile(filename) { | ||
| try { | ||
| if (fs.existsSync(filename)) { | ||
| return fs.readFileSync(filename, 'utf8'); | ||
| } | ||
| return ''; | ||
| } catch (error) { | ||
| return `Error reading ${filename}: ${error.message}`; | ||
| } | ||
| } | ||
| // Helper function to extract error context from output | ||
| function extractErrorContext(output, maxLines = 10) { | ||
| if (!output) return 'No output captured'; | ||
| const lines = output.split('\n'); | ||
| const errorLines = []; | ||
| let capturing = false; | ||
| for (let i = 0; i < lines.length && errorLines.length < maxLines; i++) { | ||
| const line = lines[i]; | ||
| // Look for error indicators | ||
| if (line.includes('FAIL') || line.includes('Error:') || line.includes('Failed:') || | ||
| line.includes('● ') || line.includes('✕') || line.includes('ERRORS:')) { | ||
| capturing = true; | ||
| } | ||
| if (capturing) { | ||
| errorLines.push(line); | ||
| // Stop capturing after finding summary or next test | ||
| if (line.includes('Test Suites:') || line.includes('Tests:')) { | ||
| break; | ||
| } | ||
| } | ||
| } | ||
| return errorLines.length > 0 ? errorLines.slice(0, maxLines).join('\n') : 'No specific error details captured'; | ||
| } | ||
| // Get step outcomes | ||
| const outcomes = { | ||
| lint: '${{ steps.lint.outcome }}', | ||
| build: '${{ steps.build.outcome }}', | ||
| coverage: '${{ steps.coverage.outcome }}', | ||
| fmlCompilation: '${{ steps.fml-compilation.outcome }}', | ||
| fmlExecution: '${{ steps.fml-execution.outcome }}', | ||
| fhirApi: '${{ steps.fhir-api.outcome }}', | ||
| validationCore: '${{ steps.validation-core.outcome }}' | ||
| }; | ||
| // Read output files | ||
| const outputs = { | ||
| lint: readOutputFile('lint-output.txt'), | ||
| build: readOutputFile('build-output.txt'), | ||
| coverage: readOutputFile('coverage-output.txt'), | ||
| fmlCompilation: readOutputFile('fml-compilation-output.txt'), | ||
| fmlExecution: readOutputFile('fml-execution-output.txt'), | ||
| fhirApi: readOutputFile('fhir-api-output.txt'), | ||
| validationCore: readOutputFile('validation-core-output.txt') | ||
| }; | ||
| // Extract test summary from coverage output | ||
| const coverageOutput = outputs.coverage; | ||
| let totalTests = 'unknown'; | ||
| let totalSuites = 'unknown'; | ||
| const testMatch = coverageOutput.match(/Tests:\s*(\d+\s+\w+)/); | ||
| const suiteMatch = coverageOutput.match(/Test Suites:\s*(\d+\s+\w+)/); | ||
| if (testMatch) totalTests = testMatch[1]; | ||
| if (suiteMatch) totalSuites = suiteMatch[1]; | ||
| // Build QA table | ||
| let qaTable = `## QA Report Summary - Node.js ${{ matrix.node-version }} | ||
| | Test Category | Status | Details | Error Context | | ||
| |---------------|--------|---------|---------------|`; | ||
| // Build Status | ||
| if (outcomes.build === 'success') { | ||
| qaTable += `\n| Build | ✅ Passed | TypeScript compilation successful | - |`; | ||
| } else { | ||
| const errorContext = extractErrorContext(outputs.build, 5); | ||
| qaTable += `\n| Build | ❌ Failed | TypeScript compilation failed | \`\`\`\n${errorContext}\n\`\`\` |`; | ||
| } | ||
| // Linting Status | ||
| if (outcomes.lint === 'success') { | ||
| qaTable += `\n| Linting | ✅ Passed | ESLint validation successful | - |`; | ||
| } else { | ||
| const errorContext = extractErrorContext(outputs.lint, 5); | ||
| qaTable += `\n| Linting | ❌ Failed | ESLint validation failed | \`\`\`\n${errorContext}\n\`\`\` |`; | ||
| } | ||
| // FML Compilation Tests | ||
| if (outcomes.fmlCompilation === 'success') { | ||
| qaTable += `\n| FML Compilation | ✅ Passed | FML parsing and compilation tests | - |`; | ||
| } else { | ||
| const errorContext = extractErrorContext(outputs.fmlCompilation, 8); | ||
| qaTable += `\n| FML Compilation | ❌ Failed | FML parsing and compilation tests | \`\`\`\n${errorContext}\n\`\`\` |`; | ||
| } | ||
| // FML Execution Tests | ||
| if (outcomes.fmlExecution === 'success') { | ||
| qaTable += `\n| FML Execution | ✅ Passed | StructureMap execution and FHIRPath tests | - |`; | ||
| } else { | ||
| const errorContext = extractErrorContext(outputs.fmlExecution, 8); | ||
| qaTable += `\n| FML Execution | ❌ Failed | StructureMap execution and FHIRPath tests | \`\`\`\n${errorContext}\n\`\`\` |`; | ||
| } | ||
| // FHIR API Tests | ||
| if (outcomes.fhirApi === 'success') { | ||
| qaTable += `\n| FHIR API | ✅ Passed | REST API endpoints and CRUD operations | - |`; | ||
| } else { | ||
| const errorContext = extractErrorContext(outputs.fhirApi, 8); | ||
| qaTable += `\n| FHIR API | ❌ Failed | REST API endpoints and CRUD operations | \`\`\`\n${errorContext}\n\`\`\` |`; | ||
| } | ||
| // Validation & Core Tests | ||
| if (outcomes.validationCore === 'success') { | ||
| qaTable += `\n| Validation & Core | ✅ Passed | Input validation and core library functions | - |`; | ||
| } else { | ||
| const errorContext = extractErrorContext(outputs.validationCore, 8); | ||
| qaTable += `\n| Validation & Core | ❌ Failed | Input validation and core library functions | \`\`\`\n${errorContext}\n\`\`\` |`; | ||
| } | ||
| // Overall Summary | ||
| const overallStatus = Object.values(outcomes).every(outcome => outcome === 'success') ? '✅ All QA checks passed' : '❌ Some QA checks failed'; | ||
| qaTable += ` | ||
| ### Summary | ||
| - **Node.js Version:** ${{ matrix.node-version }} | ||
| - **Total Tests:** ${totalTests} | ||
| - **Test Suites:** ${totalSuites} | ||
| - **Overall Status:** ${overallStatus} | ||
| <details> | ||
| <summary>🔍 View Full Test Output</summary> | ||
| **Coverage Output:** | ||
| \`\`\` | ||
| ${outputs.coverage.split('\n').slice(-20).join('\n')} | ||
| \`\`\` | ||
| </details>`; | ||
| // Post comment to PR if this is a pull request | ||
| if (context.eventName === 'pull_request') { | ||
| try { | ||
| // Check if a comment already exists from this workflow | ||
| const { data: comments } = await github.rest.issues.listComments({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| issue_number: context.issue.number, | ||
| }); | ||
| const existingComment = comments.find(comment => | ||
| comment.user.login === 'github-actions[bot]' && | ||
| comment.body.includes(`QA Report Summary - Node.js ${{ matrix.node-version }}`) | ||
| ); | ||
| if (existingComment) { | ||
| // Update existing comment | ||
| await github.rest.issues.updateComment({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| comment_id: existingComment.id, | ||
| body: qaTable | ||
| }); | ||
| console.log('Updated existing QA report comment'); | ||
| } else { | ||
| // Create new comment | ||
| await github.rest.issues.createComment({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| issue_number: context.issue.number, | ||
| body: qaTable | ||
| }); | ||
| console.log('Created new QA report comment'); | ||
| } | ||
| } catch (error) { | ||
| console.error('Error posting PR comment:', error); | ||
| // Fall back to step summary | ||
| core.summary.addRaw(qaTable).write(); | ||
| } | ||
| } else { | ||
| // For push events, just write to step summary | ||
| core.summary.addRaw(qaTable).write(); | ||
| } | ||
| - name: Upload test artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: test-results-node-${{ matrix.node-version }} | ||
| path: | | ||
| coverage/ | ||
| dist/ | ||
| *-output.txt | ||
