workspace-tests.yaml

name: Workspace Tests

on:
  workflow_run:
    workflows: [Pull Request Actions]
    types: [completed]

jobs:
  resolve:
    if: ${{ github.event.workflow_run.conclusion == 'success' }}
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      statuses: write
    env:
      TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
    outputs:
      workspace: ${{ steps.meta.outputs.workspace }}
      overlayBranch: ${{ steps.meta.outputs.overlayBranch }}
      overlayRepo: ${{ steps.meta.outputs.overlayRepo }}
      overlayCommit: ${{ steps.meta.outputs.overlayCommit }}
      prNumber: ${{ steps.meta.outputs.pr }}
      publishedExports: ${{ steps.meta.outputs.publishedExports }}
    steps:
      # the artifact of the current run contains PR number
      - name: Download context artifact 
        uses: dawidd6/action-download-artifact@v6
        with:
          name: context-${{ env.TRIGGERING_RUN_ID }}
          run_id: ${{ env.TRIGGERING_RUN_ID }}
          path: ./context
          if_no_artifact_found: fail

      - name: Check context for workspace and PR
        id: context
        run: |
          workspace=$(jq -r '.workspace // ""' ./context/meta.json)
          pr=$(jq -r '.pr // ""' ./context/meta.json)
          echo "workspace=$workspace" >> $GITHUB_OUTPUT
          echo "pr=$pr" >> $GITHUB_OUTPUT
          if [ -z "$workspace" ]; then
            echo "No workspace in context - skipping tests"
          fi

      - name: Download published-exports artifact for this PR
        if: steps.context.outputs.workspace != '' && steps.context.outputs.pr != ''
        uses: dawidd6/action-download-artifact@v6
        with:
          name: published-exports-pr-${{ steps.context.outputs.pr }}
          workflow: pr-actions.yaml
          workflow_conclusion: success
          workflow_search: true
          search_artifacts: true
          allow_forks: true
          if_no_artifact_found: fail

      - name: Verify published-exports artifact belongs to triggering PR
        if: steps.context.outputs.workspace != ''
        run: |
          triggering_pr=$(jq -r .pr ./context/meta.json)
          artifact_pr=$(jq -r .pr ./meta.json)
          if [[ "$triggering_pr" != "$artifact_pr" ]]; then
            echo "::error::Mismatch: published-exports artifact does not belong to triggering PR"
            echo "Triggering PR: $triggering_pr"
            echo "Published-exports artifact PR: $artifact_pr"
            exit 1
          fi

      - name: Read artifact metadata
        id: meta
        run: |
          workspace="${{ steps.context.outputs.workspace }}"
          if [[ -n "$workspace" ]]; then
            meta_file="./meta.json"
            exports_file="./published-exports.txt"
          else
            meta_file="./context/meta.json"
            exports_file=""
          fi
          {
            echo "workspace=$workspace"
            echo "overlayBranch=$(jq -r .overlayBranch "$meta_file")"
            echo "overlayRepo=$(jq -r .overlayRepo "$meta_file")"
            echo "overlayCommit=$(jq -r .overlayCommit "$meta_file")"
            echo "pr=$(jq -r '.pr // "null"' "$meta_file")"
            if [[ -n "$workspace" ]] && [[ -f "$exports_file" ]]; then
              echo "publishedExports<<EOF"
              cat "$exports_file"
              echo "EOF"
            else
              echo "publishedExports="
            fi
          } >> $GITHUB_OUTPUT

      - name: Debug resolved metadata
        env:
          WORKSPACE: ${{ steps.meta.outputs.workspace }}
          PR: ${{ steps.meta.outputs.pr }}
          OVERLAY_SHA: ${{ steps.meta.outputs.overlayCommit }}
        run: |
          echo "Workspace: $WORKSPACE"
          echo "PR: $PR, Overlay SHA: $OVERLAY_SHA"

      - name: Set pending commit status
        if: steps.meta.outputs.pr != 'null' && steps.meta.outputs.pr != ''
        uses: actions/github-script@v7
        env:
          OVERLAY_COMMIT: ${{ steps.meta.outputs.overlayCommit }}
          PR_NUMBER: ${{ steps.meta.outputs.pr }}
        with:
          script: |
            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const overlayCommit = process.env.OVERLAY_COMMIT;
            const pr = Number(process.env.PR_NUMBER);
            if (!pr || !overlayCommit) {
              console.log('Missing PR or commit; skipping pending status');
              return;
            }
            
            await github.rest.repos.createCommitStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              sha: overlayCommit,
              description: 'Workspace Tests',
              state: 'pending',
              target_url: runUrl,
              context: 'test',
            });
            console.log(`Set pending status on ${overlayCommit}`);

  prepare-test-config:
    needs: resolve
    if: ${{ needs.resolve.outputs.workspace != '' }}
    runs-on: ubuntu-latest
    outputs:
      plugins_metadata_complete: ${{ steps.build-dynamic-plugins.outputs.plugins_metadata_complete }}
      skip_tests_missing_env: ${{ steps.build-dynamic-plugins.outputs.skip_tests_missing_env }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.resolve.outputs.overlayBranch }}
          repository: ${{ needs.resolve.outputs.overlayRepo }}

      - name: Build dynamic-plugins.test.yaml
        id: build-dynamic-plugins
        env:
          WORKSPACE_PATH: ${{ needs.resolve.outputs.workspace }}
          PUBLISHED_EXPORTS: ${{ needs.resolve.outputs.publishedExports }}
        run: |
          PLUGINS_FOUND=0
          PLUGINS_SKIPPED_MISSING_ENV=0
          TEST_PLUGINS_SKIPPED=0
          TOTAL_PLUGINS=0
          PLUGINS_METADATA_COMPLETE="false"
          SKIP_TESTS_MISSING_ENV="false"

          if [ -z "$PUBLISHED_EXPORTS" ]; then
            echo "No published exports provided."
            echo "plugins_metadata_complete=$PLUGINS_METADATA_COMPLETE" >> "$GITHUB_OUTPUT"
            echo "skip_tests_missing_env=$SKIP_TESTS_MISSING_ENV" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # Count total plugins from exports
          for export in $PUBLISHED_EXPORTS; do
            TOTAL_PLUGINS=$((TOTAL_PLUGINS + 1))
          done

          OUT_DIR="$WORKSPACE_PATH/tests"
          OUT_FILE="$OUT_DIR/dynamic-plugins.test.yaml"
          mkdir -p "$OUT_DIR"

          # Build map of <stripped packageName> -> metadata file path
          declare -A META_MAP
          for file in "$WORKSPACE_PATH"/metadata/*.yaml; do
            [ -e "$file" ] || continue
            pkg=$(yq -r '.spec.packageName // ""' "$file")
            if [ -n "$pkg" ] && [ "$pkg" != "null" ]; then
              stripped=$(echo "$pkg" | sed 's|^@||; s|/|-|')
              META_MAP["$stripped"]="$file"
            fi
          done

          # Always include the root-level default config
          ROOT_CONFIG="$GITHUB_WORKSPACE/tests/app-config.yaml"
          [ -f "$ROOT_CONFIG" ] && cp "$ROOT_CONFIG" "$OUT_DIR/app-config.yaml"
          
          # Read workspace-wide test.env if it exists
          WORKSPACE_ENV_FILE="$WORKSPACE_PATH/tests/test.env"
          WORKSPACE_ENV_CONTENT=""
          if [ -f "$WORKSPACE_ENV_FILE" ]; then
            echo "Found workspace test.env file: $WORKSPACE_ENV_FILE"
            WORKSPACE_ENV_CONTENT=$(cat "$WORKSPACE_ENV_FILE")
          else
            echo "No workspace test.env file found at: $WORKSPACE_ENV_FILE"
          fi

          # Start the resulting YAML file
          echo "plugins:" > "$OUT_FILE"

          # For each published export, extract plugin name and read its metadata
          # Expected export format: ghcr.io/<repo_path>/<plugin_name>:<tag>
          for export in $PUBLISHED_EXPORTS; do
            if [[ "$export" =~ ^ghcr\.io/(.+):([^[:space:]]+)$ ]]; then
              IMAGE_PATH_AND_PLUGIN="${BASH_REMATCH[1]}"   # <repo_path>/<plugin_name>
              NEW_TAG="${BASH_REMATCH[2]}"                 # <tag>
              PLUGIN_NAME="${IMAGE_PATH_AND_PLUGIN##*/}"
              METADATA_FILE="${META_MAP[$PLUGIN_NAME]}"
              if [ -z "$METADATA_FILE" ]; then
                # if the name of the plugin ends with -test, it is a test plugin without metadata:
                # skip it without cancelling the test workflow
                if [[ "$PLUGIN_NAME" =~ -test$ ]]; then
                  echo "Plugin $PLUGIN_NAME is a test plugin without metadata, skipping this individual plugin test"
                  TEST_PLUGINS_SKIPPED=$((TEST_PLUGINS_SKIPPED + 1))
                else
                  echo "Metadata mapping not found for $PLUGIN_NAME: test workflow will be skipped"
                fi
                continue
              fi

              PACKAGE_NAME=$(yq -r '.spec.packageName' "$METADATA_FILE")
              if [ -z "$PACKAGE_NAME" ] || [ "$PACKAGE_NAME" = "null" ]; then
                echo "spec.packageName not found in $METADATA_FILE, skipping"
                continue
              fi

              # First appConfigExamples item is used for testing
              CONFIG_CONTENT=$(yq -o=yaml '.spec.appConfigExamples[0].content' "$METADATA_FILE" 2>/dev/null || echo "")
              if [ -z "$CONFIG_CONTENT" ] || [ "$CONFIG_CONTENT" = "null" ]; then
                echo "spec.appConfigExamples[0].content not found in $METADATA_FILE: assuming empty config"
                CONFIG_CONTENT=""
              fi

              ENV_VARS=$(echo "$CONFIG_CONTENT" | yq -o=yaml '.dynamicPlugins' 2>/dev/null | grep -oE '\$\{[A-Z_][A-Z0-9_]*\}|\$[A-Z_][A-Z0-9_]*' | sed 's/\${//; s/}//; s/^\$//' | sort -u || true)
              
              if [ -n "$ENV_VARS" ]; then
                # Config contains environment variables
                if [ -z "$WORKSPACE_ENV_CONTENT" ]; then
                  echo "  ::warning::Config for $PLUGIN_NAME contains environment variables but workspace test.env is missing. Tests will be skipped."
                  SKIP_TESTS_MISSING_ENV="true"
                  PLUGINS_SKIPPED_MISSING_ENV=$((PLUGINS_SKIPPED_MISSING_ENV + 1))
                  continue
                else
                  # Validate all ENVs are present in merged env file
                  MISSING_ENVS=()
                  while IFS= read -r env_var; do
                    [ -z "$env_var" ] && continue
                    if ! echo "$WORKSPACE_ENV_CONTENT" | grep -qE "^[[:space:]]*${env_var}[[:space:]]*="; then
                      MISSING_ENVS+=("$env_var")
                    fi
                  done <<< "$ENV_VARS"
                  
                  if [ ${#MISSING_ENVS[@]} -gt 0 ]; then
                    echo "  ::error::Environment variables missing from test.env: ${MISSING_ENVS[*]}"
                    echo "  Config for $PLUGIN_NAME references these environment variables but they are not defined in $WORKSPACE_ENV_FILE."
                    exit 1
                  fi
                fi
              fi
              # If no ENVs in config, continue regardless of env file existence

              STRIPPED=$(echo "$PACKAGE_NAME" | sed 's|^@||; s|/|-|')
              echo "- package: \"oci://ghcr.io/${IMAGE_PATH_AND_PLUGIN}:${NEW_TAG}!${STRIPPED}\"" >> "$OUT_FILE"
              echo "  disabled: false" >> "$OUT_FILE"
              if [ -n "$CONFIG_CONTENT" ]; then
                echo "  pluginConfig:" >> "$OUT_FILE"
                echo "$CONFIG_CONTENT" | sed 's/^/    /' >> "$OUT_FILE"
              fi
              PLUGINS_FOUND=$((PLUGINS_FOUND + 1))
            else
              echo "Export did not match expected format, skipping: $export"
            fi
          done

          if [ "$PLUGINS_FOUND" -eq 0 ]; then
            echo "[]" >> "$OUT_FILE"
          fi

          # Check if all plugins were found (including those skipped due to missing env)
          TOTAL_PROCESSED=$((PLUGINS_FOUND + PLUGINS_SKIPPED_MISSING_ENV + TEST_PLUGINS_SKIPPED))
          echo "Plugins: $PLUGINS_FOUND/$TOTAL_PLUGINS processed successfully"
          [ "${PLUGINS_SKIPPED_MISSING_ENV:-0}" -gt 0 ] && echo "Skipped $PLUGINS_SKIPPED_MISSING_ENV (missing test.env)"

          if [ "$TOTAL_PROCESSED" -eq "$TOTAL_PLUGINS" ] && [ "$TOTAL_PLUGINS" -gt 0 ]; then
            PLUGINS_METADATA_COMPLETE="true"
          fi

          echo "plugins_metadata_complete=$PLUGINS_METADATA_COMPLETE" >> "$GITHUB_OUTPUT"
          echo "skip_tests_missing_env=$SKIP_TESTS_MISSING_ENV" >> "$GITHUB_OUTPUT"

      - name: Upload integration-test artefact
        uses: actions/upload-artifact@v4
        with:
          name: integration-test-artifacts
          path: ${{ format('{0}/tests', needs.resolve.outputs.workspace) }}
          if-no-files-found: error

  integration-tests:
    needs:
      - resolve
      - prepare-test-config
    if: ${{ needs.prepare-test-config.outputs.plugins_metadata_complete == 'true' && needs.prepare-test-config.outputs.skip_tests_missing_env != 'true' }}
    uses: ./.github/workflows/run-plugin-integration-tests.yaml

  add-skipped-test-comment:
    if: ${{ always() && needs.resolve.outputs.prNumber != 'null' && needs.resolve.outputs.prNumber != '' && (needs.resolve.outputs.workspace == '' || (needs.prepare-test-config.result != 'skipped' && (needs.prepare-test-config.outputs.plugins_metadata_complete != 'true' || needs.prepare-test-config.outputs.skip_tests_missing_env == 'true'))) }}
    needs:
      - resolve
      - prepare-test-config
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
      issues: write
    steps:
      - name: Post skipped-test comment
        uses: actions/github-script@v7
        env:
          INPUT_WORKSPACE: ${{ needs.resolve.outputs.workspace }}
          INPUT_PLUGINS_METADATA_COMPLETE: ${{ needs.prepare-test-config.outputs.plugins_metadata_complete || '' }}
          INPUT_SKIP_TESTS_MISSING_ENV: ${{ needs.prepare-test-config.outputs.skip_tests_missing_env || '' }}
          INPUT_PR_NUMBER: ${{ needs.resolve.outputs.prNumber }}
        with:
          script: |
            const pr = Number(core.getInput('pr_number') || '0');
            if (!pr) {
              console.log('No PR associated; skipping');
              return;
            }
            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const workspace = core.getInput('workspace');
            const pluginsMetadataComplete = core.getInput('plugins_metadata_complete') === 'true';
            const skipTestsMissingEnv = core.getInput('skip_tests_missing_env') === 'true';
            let body = `:warning: \n[Test workflow](${runUrl})`;

            if (!workspace || workspace === '') {
              body += ' skipped: PR doesn\'t touch exactly one workspace.\n';
            } else if (skipTestsMissingEnv) {
              body += ' skipped: missing workspace `tests/test.env` file.\n';
            } else if (!pluginsMetadataComplete) {
              body += ' skipped: missing plugin metadata files (`<workspace>/metadata/*.yaml`).\n';
            } else {
              body += ' skipped for an unknown reason. Check workflow run for details.\n';
            }
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: pr,
              body,
            });

  add-test-result-comment:
    if: ${{ always() && needs.prepare-test-config.outputs.plugins_metadata_complete == 'true' && needs.prepare-test-config.outputs.skip_tests_missing_env != 'true' }}
    needs:
      - resolve
      - prepare-test-config
      - integration-tests
    concurrency:
      group: addTestResultComment-${{ github.ref_name }}
      cancel-in-progress: true
    permissions:
      contents: read
      pull-requests: write
      issues: write
      statuses: write
    runs-on: ubuntu-latest
    steps:
      - name: Post test result comment and status
        uses: actions/github-script@v7
        env:
          INTEGRATION_TESTS_RESULT: ${{ needs.integration-tests.result }}
          SUCCESS: ${{ needs.integration-tests.outputs.success }}
          FAILED_PLUGINS: ${{ needs.integration-tests.outputs.failed-plugins }}
          ERROR_LOGS: ${{ needs.integration-tests.outputs.error-logs }}
          PR_NUMBER: ${{ needs.resolve.outputs.prNumber }}
          OVERLAY_COMMIT: ${{ needs.resolve.outputs.overlayCommit }}
        with:
          script: |
            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const integrationTestsResult = process.env.INTEGRATION_TESTS_RESULT || '';
            const successOutput = process.env.SUCCESS;
            const failed = (process.env.FAILED_PLUGINS || '').trim();
            const errorLogs = (process.env.ERROR_LOGS || '').trim();
            const pr = Number(process.env.PR_NUMBER);
            const overlayCommit = process.env.OVERLAY_COMMIT;
            
            if (!pr) {
              console.log('No PR associated; skipping');
              return;
            }
            
            const success = integrationTestsResult === 'success' && successOutput === 'true';
            let failureReason = '';
            
            if (!success) {
              switch (integrationTestsResult) {
                case 'failure':
                  failureReason = '\n\n:warning: Integration tests failed. Check the workflow logs for details.';
                  break;
                case 'cancelled':
                  failureReason = '\n\n:warning: Integration tests were cancelled.';
                  break;
                case 'timeout':
                  failureReason = '\n\n:warning: Integration tests timed out.';
                  break;
                default:
                  failureReason = `\n\n:warning: Integration tests ended in an unexpected state: ${integrationTestsResult}. Check the workflow logs for details.`;
              }
            }
            
            // Get current PR head SHA
            const { data: prData } = await github.rest.pulls.get({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: pr,
            });
            
            // Use PR head if different from overlayCommit (re-test), else use overlayCommit (immediate publish)
            const sha = prData.head.sha !== overlayCommit ? prData.head.sha : overlayCommit;
            console.log(`Status SHA: ${sha} (PR head: ${prData.head.sha}, overlay: ${overlayCommit})`);
            
            await github.rest.repos.createCommitStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              sha,
              description: 'Workspace Tests',
              state: success ? 'success' : 'failure',
              target_url: runUrl,
              context: 'test',
            });
            
            let body;
            if (success) {
              body = `:white_check_mark: [Test workflow](${runUrl}) passed. All plugins loaded successfully.\n`;
            } else {
              body = `:x: \n[Test workflow](${runUrl}) failed.`;
              body += failureReason;
              if (failed) {
                body += `\n\nThese plugins failed to load:\n${failed}`;
              }
              if (errorLogs) {
                body += `\n\n<details><summary>Error logs from container</summary>\n\n\`\`\`\n${errorLogs}\n\`\`\`\n\n</details>`;
              }
            }
            
            await github.rest.issues.createComment({
              issue_number: pr,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body,
            });