Workspace Tests #1402
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Workspace Tests | |
| on: | |
| workflow_run: | |
| workflows: [Pull Request Actions] | |
| types: [completed] | |
| jobs: | |
| resolve: | |
| if: ${{ github.event.workflow_run.conclusion == 'success' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| contents: read | |
| statuses: write | |
| env: | |
| TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }} | |
| outputs: | |
| workspace: ${{ steps.meta.outputs.workspace }} | |
| overlayBranch: ${{ steps.meta.outputs.overlayBranch }} | |
| overlayRepo: ${{ steps.meta.outputs.overlayRepo }} | |
| overlayCommit: ${{ steps.meta.outputs.overlayCommit }} | |
| prNumber: ${{ steps.meta.outputs.pr }} | |
| publishedExports: ${{ steps.meta.outputs.publishedExports }} | |
| steps: | |
| # the artifact of the current run contains PR number | |
| - name: Download context artifact | |
| uses: dawidd6/action-download-artifact@v6 | |
| with: | |
| name: context-${{ env.TRIGGERING_RUN_ID }} | |
| run_id: ${{ env.TRIGGERING_RUN_ID }} | |
| path: ./context | |
| if_no_artifact_found: fail | |
| - name: Check context for workspace and PR | |
| id: context | |
| run: | | |
| workspace=$(jq -r '.workspace // ""' ./context/meta.json) | |
| pr=$(jq -r '.pr // ""' ./context/meta.json) | |
| echo "workspace=$workspace" >> $GITHUB_OUTPUT | |
| echo "pr=$pr" >> $GITHUB_OUTPUT | |
| if [ -z "$workspace" ]; then | |
| echo "No workspace in context - skipping tests" | |
| fi | |
| - name: Download published-exports artifact for this PR | |
| if: steps.context.outputs.workspace != '' && steps.context.outputs.pr != '' | |
| uses: dawidd6/action-download-artifact@v6 | |
| with: | |
| name: published-exports-pr-${{ steps.context.outputs.pr }} | |
| workflow: pr-actions.yaml | |
| workflow_conclusion: success | |
| workflow_search: true | |
| search_artifacts: true | |
| allow_forks: true | |
| if_no_artifact_found: fail | |
| - name: Verify published-exports artifact belongs to triggering PR | |
| if: steps.context.outputs.workspace != '' | |
| run: | | |
| triggering_pr=$(jq -r .pr ./context/meta.json) | |
| artifact_pr=$(jq -r .pr ./meta.json) | |
| if [[ "$triggering_pr" != "$artifact_pr" ]]; then | |
| echo "::error::Mismatch: published-exports artifact does not belong to triggering PR" | |
| echo "Triggering PR: $triggering_pr" | |
| echo "Published-exports artifact PR: $artifact_pr" | |
| exit 1 | |
| fi | |
| - name: Read artifact metadata | |
| id: meta | |
| run: | | |
| workspace="${{ steps.context.outputs.workspace }}" | |
| if [[ -n "$workspace" ]]; then | |
| meta_file="./meta.json" | |
| exports_file="./published-exports.txt" | |
| else | |
| meta_file="./context/meta.json" | |
| exports_file="" | |
| fi | |
| { | |
| echo "workspace=$workspace" | |
| echo "overlayBranch=$(jq -r .overlayBranch "$meta_file")" | |
| echo "overlayRepo=$(jq -r .overlayRepo "$meta_file")" | |
| echo "overlayCommit=$(jq -r .overlayCommit "$meta_file")" | |
| echo "pr=$(jq -r '.pr // "null"' "$meta_file")" | |
| if [[ -n "$workspace" ]] && [[ -f "$exports_file" ]]; then | |
| echo "publishedExports<<EOF" | |
| cat "$exports_file" | |
| echo "EOF" | |
| else | |
| echo "publishedExports=" | |
| fi | |
| } >> $GITHUB_OUTPUT | |
| - name: Debug resolved metadata | |
| env: | |
| WORKSPACE: ${{ steps.meta.outputs.workspace }} | |
| PR: ${{ steps.meta.outputs.pr }} | |
| OVERLAY_SHA: ${{ steps.meta.outputs.overlayCommit }} | |
| run: | | |
| echo "Workspace: $WORKSPACE" | |
| echo "PR: $PR, Overlay SHA: $OVERLAY_SHA" | |
| - name: Set pending commit status | |
| if: steps.meta.outputs.pr != 'null' && steps.meta.outputs.pr != '' | |
| uses: actions/github-script@v7 | |
| env: | |
| OVERLAY_COMMIT: ${{ steps.meta.outputs.overlayCommit }} | |
| PR_NUMBER: ${{ steps.meta.outputs.pr }} | |
| with: | |
| script: | | |
| const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const overlayCommit = process.env.OVERLAY_COMMIT; | |
| const pr = Number(process.env.PR_NUMBER); | |
| if (!pr || !overlayCommit) { | |
| console.log('Missing PR or commit; skipping pending status'); | |
| return; | |
| } | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha: overlayCommit, | |
| description: 'Workspace Tests', | |
| state: 'pending', | |
| target_url: runUrl, | |
| context: 'test', | |
| }); | |
| console.log(`Set pending status on ${overlayCommit}`); | |
| prepare-test-config: | |
| needs: resolve | |
| if: ${{ needs.resolve.outputs.workspace != '' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| plugins_metadata_complete: ${{ steps.build-dynamic-plugins.outputs.plugins_metadata_complete }} | |
| skip_tests_missing_env: ${{ steps.build-dynamic-plugins.outputs.skip_tests_missing_env }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.resolve.outputs.overlayBranch }} | |
| repository: ${{ needs.resolve.outputs.overlayRepo }} | |
| - name: Build dynamic-plugins.test.yaml | |
| id: build-dynamic-plugins | |
| env: | |
| WORKSPACE_PATH: ${{ needs.resolve.outputs.workspace }} | |
| PUBLISHED_EXPORTS: ${{ needs.resolve.outputs.publishedExports }} | |
| run: | | |
| PLUGINS_FOUND=0 | |
| PLUGINS_SKIPPED_MISSING_ENV=0 | |
| TEST_PLUGINS_SKIPPED=0 | |
| TOTAL_PLUGINS=0 | |
| PLUGINS_METADATA_COMPLETE="false" | |
| SKIP_TESTS_MISSING_ENV="false" | |
| if [ -z "$PUBLISHED_EXPORTS" ]; then | |
| echo "No published exports provided." | |
| echo "plugins_metadata_complete=$PLUGINS_METADATA_COMPLETE" >> "$GITHUB_OUTPUT" | |
| echo "skip_tests_missing_env=$SKIP_TESTS_MISSING_ENV" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| # Count total plugins from exports | |
| for export in $PUBLISHED_EXPORTS; do | |
| TOTAL_PLUGINS=$((TOTAL_PLUGINS + 1)) | |
| done | |
| OUT_DIR="$WORKSPACE_PATH/tests" | |
| OUT_FILE="$OUT_DIR/dynamic-plugins.test.yaml" | |
| mkdir -p "$OUT_DIR" | |
| # Build map of <stripped packageName> -> metadata file path | |
| declare -A META_MAP | |
| for file in "$WORKSPACE_PATH"/metadata/*.yaml; do | |
| [ -e "$file" ] || continue | |
| pkg=$(yq -r '.spec.packageName // ""' "$file") | |
| if [ -n "$pkg" ] && [ "$pkg" != "null" ]; then | |
| stripped=$(echo "$pkg" | sed 's|^@||; s|/|-|') | |
| META_MAP["$stripped"]="$file" | |
| fi | |
| done | |
| # Always include the root-level default config | |
| ROOT_CONFIG="$GITHUB_WORKSPACE/tests/app-config.yaml" | |
| [ -f "$ROOT_CONFIG" ] && cp "$ROOT_CONFIG" "$OUT_DIR/app-config.yaml" | |
| # Read workspace-wide test.env if it exists | |
| WORKSPACE_ENV_FILE="$WORKSPACE_PATH/tests/test.env" | |
| WORKSPACE_ENV_CONTENT="" | |
| if [ -f "$WORKSPACE_ENV_FILE" ]; then | |
| echo "Found workspace test.env file: $WORKSPACE_ENV_FILE" | |
| WORKSPACE_ENV_CONTENT=$(cat "$WORKSPACE_ENV_FILE") | |
| else | |
| echo "No workspace test.env file found at: $WORKSPACE_ENV_FILE" | |
| fi | |
| # Start the resulting YAML file | |
| echo "plugins:" > "$OUT_FILE" | |
| # For each published export, extract plugin name and read its metadata | |
| # Expected export format: ghcr.io/<repo_path>/<plugin_name>:<tag> | |
| for export in $PUBLISHED_EXPORTS; do | |
| if [[ "$export" =~ ^ghcr\.io/(.+):([^[:space:]]+)$ ]]; then | |
| IMAGE_PATH_AND_PLUGIN="${BASH_REMATCH[1]}" # <repo_path>/<plugin_name> | |
| NEW_TAG="${BASH_REMATCH[2]}" # <tag> | |
| PLUGIN_NAME="${IMAGE_PATH_AND_PLUGIN##*/}" | |
| METADATA_FILE="${META_MAP[$PLUGIN_NAME]}" | |
| if [ -z "$METADATA_FILE" ]; then | |
| # if the name of the plugin ends with -test, it is a test plugin without metadata: | |
| # skip it without cancelling the test workflow | |
| if [[ "$PLUGIN_NAME" =~ -test$ ]]; then | |
| echo "Plugin $PLUGIN_NAME is a test plugin without metadata, skipping this individual plugin test" | |
| TEST_PLUGINS_SKIPPED=$((TEST_PLUGINS_SKIPPED + 1)) | |
| else | |
| echo "Metadata mapping not found for $PLUGIN_NAME: test workflow will be skipped" | |
| fi | |
| continue | |
| fi | |
| PACKAGE_NAME=$(yq -r '.spec.packageName' "$METADATA_FILE") | |
| if [ -z "$PACKAGE_NAME" ] || [ "$PACKAGE_NAME" = "null" ]; then | |
| echo "spec.packageName not found in $METADATA_FILE, skipping" | |
| continue | |
| fi | |
| # First appConfigExamples item is used for testing | |
| CONFIG_CONTENT=$(yq -o=yaml '.spec.appConfigExamples[0].content' "$METADATA_FILE" 2>/dev/null || echo "") | |
| if [ -z "$CONFIG_CONTENT" ] || [ "$CONFIG_CONTENT" = "null" ]; then | |
| echo "spec.appConfigExamples[0].content not found in $METADATA_FILE: assuming empty config" | |
| CONFIG_CONTENT="" | |
| fi | |
| ENV_VARS=$(echo "$CONFIG_CONTENT" | yq -o=yaml '.dynamicPlugins' 2>/dev/null | grep -oE '\$\{[A-Z_][A-Z0-9_]*\}|\$[A-Z_][A-Z0-9_]*' | sed 's/\${//; s/}//; s/^\$//' | sort -u || true) | |
| if [ -n "$ENV_VARS" ]; then | |
| # Config contains environment variables | |
| if [ -z "$WORKSPACE_ENV_CONTENT" ]; then | |
| echo " ::warning::Config for $PLUGIN_NAME contains environment variables but workspace test.env is missing. Tests will be skipped." | |
| SKIP_TESTS_MISSING_ENV="true" | |
| PLUGINS_SKIPPED_MISSING_ENV=$((PLUGINS_SKIPPED_MISSING_ENV + 1)) | |
| continue | |
| else | |
| # Validate all ENVs are present in merged env file | |
| MISSING_ENVS=() | |
| while IFS= read -r env_var; do | |
| [ -z "$env_var" ] && continue | |
| if ! echo "$WORKSPACE_ENV_CONTENT" | grep -qE "^[[:space:]]*${env_var}[[:space:]]*="; then | |
| MISSING_ENVS+=("$env_var") | |
| fi | |
| done <<< "$ENV_VARS" | |
| if [ ${#MISSING_ENVS[@]} -gt 0 ]; then | |
| echo " ::error::Environment variables missing from test.env: ${MISSING_ENVS[*]}" | |
| echo " Config for $PLUGIN_NAME references these environment variables but they are not defined in $WORKSPACE_ENV_FILE." | |
| exit 1 | |
| fi | |
| fi | |
| fi | |
| # If no ENVs in config, continue regardless of env file existence | |
| STRIPPED=$(echo "$PACKAGE_NAME" | sed 's|^@||; s|/|-|') | |
| echo "- package: \"oci://ghcr.io/${IMAGE_PATH_AND_PLUGIN}:${NEW_TAG}!${STRIPPED}\"" >> "$OUT_FILE" | |
| echo " disabled: false" >> "$OUT_FILE" | |
| if [ -n "$CONFIG_CONTENT" ]; then | |
| echo " pluginConfig:" >> "$OUT_FILE" | |
| echo "$CONFIG_CONTENT" | sed 's/^/ /' >> "$OUT_FILE" | |
| fi | |
| PLUGINS_FOUND=$((PLUGINS_FOUND + 1)) | |
| else | |
| echo "Export did not match expected format, skipping: $export" | |
| fi | |
| done | |
| if [ "$PLUGINS_FOUND" -eq 0 ]; then | |
| echo "[]" >> "$OUT_FILE" | |
| fi | |
| # Check if all plugins were found (including those skipped due to missing env) | |
| TOTAL_PROCESSED=$((PLUGINS_FOUND + PLUGINS_SKIPPED_MISSING_ENV + TEST_PLUGINS_SKIPPED)) | |
| echo "Plugins: $PLUGINS_FOUND/$TOTAL_PLUGINS processed successfully" | |
| [ "${PLUGINS_SKIPPED_MISSING_ENV:-0}" -gt 0 ] && echo "Skipped $PLUGINS_SKIPPED_MISSING_ENV (missing test.env)" | |
| if [ "$TOTAL_PROCESSED" -eq "$TOTAL_PLUGINS" ] && [ "$TOTAL_PLUGINS" -gt 0 ]; then | |
| PLUGINS_METADATA_COMPLETE="true" | |
| fi | |
| echo "plugins_metadata_complete=$PLUGINS_METADATA_COMPLETE" >> "$GITHUB_OUTPUT" | |
| echo "skip_tests_missing_env=$SKIP_TESTS_MISSING_ENV" >> "$GITHUB_OUTPUT" | |
| - name: Upload integration-test artefact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-test-artifacts | |
| path: ${{ format('{0}/tests', needs.resolve.outputs.workspace) }} | |
| if-no-files-found: error | |
| integration-tests: | |
| needs: | |
| - resolve | |
| - prepare-test-config | |
| if: ${{ needs.prepare-test-config.outputs.plugins_metadata_complete == 'true' && needs.prepare-test-config.outputs.skip_tests_missing_env != 'true' }} | |
| uses: ./.github/workflows/run-plugin-integration-tests.yaml | |
| add-skipped-test-comment: | |
| if: ${{ always() && needs.resolve.outputs.prNumber != 'null' && needs.resolve.outputs.prNumber != '' && (needs.resolve.outputs.workspace == '' || (needs.prepare-test-config.result != 'skipped' && (needs.prepare-test-config.outputs.plugins_metadata_complete != 'true' || needs.prepare-test-config.outputs.skip_tests_missing_env == 'true'))) }} | |
| needs: | |
| - resolve | |
| - prepare-test-config | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| issues: write | |
| steps: | |
| - name: Post skipped-test comment | |
| uses: actions/github-script@v7 | |
| env: | |
| INPUT_WORKSPACE: ${{ needs.resolve.outputs.workspace }} | |
| INPUT_PLUGINS_METADATA_COMPLETE: ${{ needs.prepare-test-config.outputs.plugins_metadata_complete || '' }} | |
| INPUT_SKIP_TESTS_MISSING_ENV: ${{ needs.prepare-test-config.outputs.skip_tests_missing_env || '' }} | |
| INPUT_PR_NUMBER: ${{ needs.resolve.outputs.prNumber }} | |
| with: | |
| script: | | |
| const pr = Number(core.getInput('pr_number') || '0'); | |
| if (!pr) { | |
| console.log('No PR associated; skipping'); | |
| return; | |
| } | |
| const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const workspace = core.getInput('workspace'); | |
| const pluginsMetadataComplete = core.getInput('plugins_metadata_complete') === 'true'; | |
| const skipTestsMissingEnv = core.getInput('skip_tests_missing_env') === 'true'; | |
| let body = `:warning: \n[Test workflow](${runUrl})`; | |
| if (!workspace || workspace === '') { | |
| body += ' skipped: PR doesn\'t touch exactly one workspace.\n'; | |
| } else if (skipTestsMissingEnv) { | |
| body += ' skipped: missing workspace `tests/test.env` file.\n'; | |
| } else if (!pluginsMetadataComplete) { | |
| body += ' skipped: missing plugin metadata files (`<workspace>/metadata/*.yaml`).\n'; | |
| } else { | |
| body += ' skipped for an unknown reason. Check workflow run for details.\n'; | |
| } | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: pr, | |
| body, | |
| }); | |
| add-test-result-comment: | |
| if: ${{ always() && needs.prepare-test-config.outputs.plugins_metadata_complete == 'true' && needs.prepare-test-config.outputs.skip_tests_missing_env != 'true' }} | |
| needs: | |
| - resolve | |
| - prepare-test-config | |
| - integration-tests | |
| concurrency: | |
| group: addTestResultComment-${{ github.ref_name }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| statuses: write | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Post test result comment and status | |
| uses: actions/github-script@v7 | |
| env: | |
| INTEGRATION_TESTS_RESULT: ${{ needs.integration-tests.result }} | |
| SUCCESS: ${{ needs.integration-tests.outputs.success }} | |
| FAILED_PLUGINS: ${{ needs.integration-tests.outputs.failed-plugins }} | |
| ERROR_LOGS: ${{ needs.integration-tests.outputs.error-logs }} | |
| PR_NUMBER: ${{ needs.resolve.outputs.prNumber }} | |
| OVERLAY_COMMIT: ${{ needs.resolve.outputs.overlayCommit }} | |
| with: | |
| script: | | |
| const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const integrationTestsResult = process.env.INTEGRATION_TESTS_RESULT || ''; | |
| const successOutput = process.env.SUCCESS; | |
| const failed = (process.env.FAILED_PLUGINS || '').trim(); | |
| const errorLogs = (process.env.ERROR_LOGS || '').trim(); | |
| const pr = Number(process.env.PR_NUMBER); | |
| const overlayCommit = process.env.OVERLAY_COMMIT; | |
| if (!pr) { | |
| console.log('No PR associated; skipping'); | |
| return; | |
| } | |
| const success = integrationTestsResult === 'success' && successOutput === 'true'; | |
| let failureReason = ''; | |
| if (!success) { | |
| switch (integrationTestsResult) { | |
| case 'failure': | |
| failureReason = '\n\n:warning: Integration tests failed. Check the workflow logs for details.'; | |
| break; | |
| case 'cancelled': | |
| failureReason = '\n\n:warning: Integration tests were cancelled.'; | |
| break; | |
| case 'timeout': | |
| failureReason = '\n\n:warning: Integration tests timed out.'; | |
| break; | |
| default: | |
| failureReason = `\n\n:warning: Integration tests ended in an unexpected state: ${integrationTestsResult}. Check the workflow logs for details.`; | |
| } | |
| } | |
| // Get current PR head SHA | |
| const { data: prData } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: pr, | |
| }); | |
| // Use PR head if different from overlayCommit (re-test), else use overlayCommit (immediate publish) | |
| const sha = prData.head.sha !== overlayCommit ? prData.head.sha : overlayCommit; | |
| console.log(`Status SHA: ${sha} (PR head: ${prData.head.sha}, overlay: ${overlayCommit})`); | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha, | |
| description: 'Workspace Tests', | |
| state: success ? 'success' : 'failure', | |
| target_url: runUrl, | |
| context: 'test', | |
| }); | |
| let body; | |
| if (success) { | |
| body = `:white_check_mark: [Test workflow](${runUrl}) passed. All plugins loaded successfully.\n`; | |
| } else { | |
| body = `:x: \n[Test workflow](${runUrl}) failed.`; | |
| body += failureReason; | |
| if (failed) { | |
| body += `\n\nThese plugins failed to load:\n${failed}`; | |
| } | |
| if (errorLogs) { | |
| body += `\n\n<details><summary>Error logs from container</summary>\n\n\`\`\`\n${errorLogs}\n\`\`\`\n\n</details>`; | |
| } | |
| } | |
| await github.rest.issues.createComment({ | |
| issue_number: pr, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body, | |
| }); |