ci: add new gha for running frontend e2e tests against specific stack

actions/internal/plugins/frontend-e2e-against-stack/.gitignore

@@ -0,0 +1,17 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+node_modules/
+
+coverage/
+dist/
+
+.*.bun-build
+
+# Editor
+.idea

actions/internal/plugins/frontend-e2e-against-stack/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## 1.0.0
+
+- Create new action frontend-e2e-against-stack to run e2e playwright tests against pre-selected stack.

actions/internal/plugins/frontend-e2e-against-stack/README.md

@@ -0,0 +1,62 @@
+# Run e2e tests from frontend plugins against specific stack
+
+This is a GitHub Action that help the execution of e2e tests on any frontend plugin that is using [Playwright](https://playwright.dev/) against specific selected stack.
+You need to define in which region the selected stack belong, the plugin from where are executed the tests and optionally which other plugins and datasources you want to provision when starting a Grafana instance.
+Also, you need to have the **playwright** configuration and the test specifications in the plugin that run the tests and the action will do the rest.
+This action use the following input parameters to run:
+
+| Name                  | Description                                                                                                        | Default           | Required |
+| --------------------- |--------------------------------------------------------------------------------------------------------------------|-------------------|----------|
+| `plugin-directory`    | Directory of the plugin, if not in the root of the repository. If provided, package-manager must also be provided. | .                 | No       |
+| `package-manager`     | The package manager to use for building the plugin                                                                 |                   | No       |
+| `npm-registry-auth`   | Whether to authenticate to the npm registry in Google Artifact Registry                                            | false             | No       |
+| `stack_slug`          | Name of the stack where you want to run the tests                                                                  |                   | Yes      |
+| `env`                 | Region of the stack where you want to run the tests                                                                |                   | Yes      |
+| `other_plugins`       | List of other plugins that you want to enable separated by comma                                                   |                   | No       |
+| `datasource_ids`      | List of data sources that you want to enable separated by comma                                                    |                   | No       |
+| `upload_report_path ` | Name of the folder where you want to store the test report                                                         | playwright-report | No       |
+| `upload_videos_path`  | Name of the folder where you want to store the test videos                                                         | playwright-videos | No       |
+| `plugin-secrets`      | A JSON string containing key-value pairs of specific plugin secrets necessary to run the tests.                    |                   | No       |
+| `grafana-ini-path`    | Path to a custom grafana.ini file to configure the Grafana instance                                                |                   | No       |
+
+## Example workflows
+
+This is an example of how you could use this action.
+
+```yml
+name: Build and Test PR
+
+on:
+  pull_request:
+
+jobs:
+  e2e-tests:
+    permissions:
+      contents: write
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Get plugin specific secrets
+        id: create-plugin-secrets
+        shell: bash
+        run: |
+          echo 'plugin-json-secrets={"MY_SECRET1": "value_secrete_1", "MY_SECRET2": "value_secret_2"}' >> "$GITHUB_OUTPUT"
+
+      - name: Run e2e cross app tests
+        id: e2e-cross-apps-tests
+        uses: grafana/plugin-ci-workflows/actions/internal/plugins/frontend-e2e-against-stack@main
+        with:
+          npm-registry-auth: "true"
+          stack_slug: "mygrafanastack"
+          env: "dev-central"
+          other_plugins: "grafana-plugin1-app,grafana-plugin2-app"
+          datasource_ids: "grafanacloud-mygrafanastack-prom,grafanacloud-mygrafanastack-logs"
+          upload_report_path: "playwright-cross-apps-report"
+          upload_videos_path: "playwright-cross-apps-videos"
+          plugin-secrets: ${{ steps.create-plugin-secrets.outputs.plugin-json-secrets }}
+          grafana-ini-path: "provisioning/custom-grafana.ini"  # Optional
+```

actions/internal/plugins/frontend-e2e-against-stack/action.yml

@@ -0,0 +1,291 @@
+name: Run e2e tests
+description: Run e2e tests against specific stack and environment
+inputs:
+  plugin-directory:
+    description: Directory of the plugin, if not in the root of the repository. If provided, package-manager must also be provided.
+    required: false
+    default: .
+
+  package-manager:
+    description: The package manager to use.
+    required: false
+    default: ""
+
+  npm-registry-auth:
+    description: |
+      Whether to authenticate to the npm registry in Google Artifact Registry.
+      If true, the root of the plugin repository must contain a `.npmrc` file.
+    required: false
+    default: "false"
+
+  stack_slug:
+    description: "Name of the stack where you want to run the tests"
+    required: true
+
+  env:
+    description: "Region of the stack where you want to run the tests"
+    required: true
+
+  other_plugins:
+    description: "List of other plugins that you want to enable separated by comma"
+    required: false
+
+  datasource_ids:
+    description: "List of data sources that you want to enable separated by comma"
+    required: false
+
+  upload_report_path:
+    description: "Name of the artifact where you want to store the test report"
+    required: false
+    default: "playwright-report"
+
+  upload_videos_path:
+    description: "Name of the artifact where you want to store the test videos"
+    required: false
+    default: "playwright-videos"
+
+  plugin-secrets:
+    description: "A JSON string containing key-value pairs of specific plugin secrets necessary to run the tests."
+    required: false
+
+  grafana-ini-path:
+    description: "Path to a custom grafana.ini file to configure the Grafana instance. Path should be relative to the plugin directory."
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repository
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        persist-credentials: false
+
+    - name: Setup Node.js environment
+      uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+      with:
+        node-version: "20"
+        cache: "yarn"
+
+    - name: Install e2e action dependencies
+      run: yarn install
+      shell: bash
+      working-directory: ${{ github.action_path }}
+
+    - name: Login to Google Cloud
+      id: gcloud-auth
+      if: inputs.npm-registry-auth == 'true'
+      uses: google-github-actions/auth@8254fb75a33b976a221574d287e93919e6a36f70 # v2.1.6
+      with:
+        token_format: access_token
+        workload_identity_provider: "projects/304398677251/locations/global/workloadIdentityPools/github/providers/github-provider"
+        service_account: "github-cloud-npm-dev-pkgs@grafanalabs-workload-identity.iam.gserviceaccount.com"
+
+    - name: NPM registry auth
+      if: inputs.npm-registry-auth == 'true'
+      shell: bash
+      working-directory: ${{ inputs.plugin-directory }}
+      run: GOOGLE_APPLICATION_CREDENTIALS=${{ env.GOOGLE_APPLICATION_CREDENTIALS }} npx google-artifactregistry-auth --credential-config ./.npmrc
+
+    - name: Install dependencies
+      shell: bash
+      working-directory: ${{ inputs.plugin-directory }}
+      run: ${{ github.action_path }}/pm.sh install
+      env:
+        PACKAGE_MANAGER: ${{ inputs.package-manager }}
+
+    - name: Build
+      shell: bash
+      working-directory: ${{ inputs.plugin-directory }}
+      run: ${{ github.action_path }}/pm.sh build
+      env:
+        PACKAGE_MANAGER: ${{ inputs.package-manager }}
+
+    # The action should end up with a dist/ folder, but if the working directory is not the root of the repo,
+    # we need to copy the dist/ folder to the root of the repo.
+    - name: Copy dist if needed
+      run: |
+        if [ "$PLUGIN_DIRECTORY" != "." ]; then
+          mkdir -p dist
+          cp -r $PLUGIN_DIRECTORY/dist/* dist/
+        fi
+      shell: bash
+      if: inputs.plugin-directory != '.'
+      env:
+        PLUGIN_DIRECTORY: ${{ inputs.plugin-directory }}
+
+    - name: Extract plugin ID from plugin.json
+      id: extract-plugin-id
+      shell: bash
+      working-directory: ${{ inputs.plugin-directory }}
+      run: |
+        if [ ! -f "src/plugin.json" ]; then
+          echo "Error: plugin.json not found at src/plugin.json"
+          exit 1
+        fi
+
+        if ! jq empty src/plugin.json 2>/dev/null; then
+          echo "Error: plugin.json contains invalid JSON"
+          exit 1
+        fi
+
+        PLUGIN_ID=$(jq -r '.id' src/plugin.json)
+
+        if [ "$PLUGIN_ID" = "null" ] || [ -z "$PLUGIN_ID" ]; then
+          echo "Error: 'id' field not found or empty in plugin.json"
+          exit 1
+        fi
+
+        echo "plugin-id=$PLUGIN_ID" >> $GITHUB_OUTPUT
+        echo "Extracted plugin ID: $PLUGIN_ID"
+
+    - name: Get common secrets
+      id: get-common-secrets
+      uses: grafana/shared-workflows/actions/get-vault-secrets@5d7e361bc7e0a183cde8afe9899fb7b596d2659b # v1.2.0
+      with:
+        common_secrets: |
+          HG_TOKEN=hg-ci:token
+
+    - name: Set plugin secrets as environment variables
+      id: set-env-vars
+      if: ${{ inputs.plugin-secrets != '' }}
+      shell: bash
+      env:
+        SECRETS_JSON: "${{ inputs.plugin-secrets }}"
+      run: |
+        echo "Parsing and setting plugin environment variables..."
+        echo "$SECRETS_JSON" | jq -r 'to_entries[] | "echo \"\(.key)=\(.value)\" >> $GITHUB_ENV"' | bash
+        echo "Plugin environment variables set."
+
+    - name: Generate provisioning
+      shell: bash
+      run: npx plop --plopfile ${{ github.action_path }}/plopfile.mjs --dest . e2e-testing-provisioning
+      working-directory: ${{ inputs.plugin-directory }}
+      env:
+        E2E_STACK_SLUG: ${{ inputs.stack_slug }}
+        E2E_ENV: ${{ inputs.env }}
+        HG_TOKEN: ${{ env.HG_TOKEN }}
+        E2E_PLUGIN_ID: ${{ steps.extract-plugin-id.outputs.plugin-id }}
+        E2E_OTHER_PLUGINS: ${{ inputs.other_plugins }}
+        E2E_DATASOURCE_IDS: ${{ inputs.datasource_ids }}
+        E2E_GRAFANA_INI_PATH: ${{ inputs.grafana-ini-path }}
+
+    - name: Start server
+      run: docker compose -f docker-compose.e2e.yaml up -d --build --quiet-pull --timestamps
+      working-directory: ${{ inputs.plugin-directory }}
+      shell: bash
+
+    - name: Install Playwright Browsers
+      run: npx playwright install chromium --with-deps
+      working-directory: ${{ inputs.plugin-directory }}
+      shell: bash
+
+    - name: Wait for Grafana to be ready
+      shell: bash
+      run: |
+        echo "Waiting for Grafana to be available..."
+        timeout=480  # 8 minutes timeout
+        start_time=$(date +%s)
+
+        # Parse expected plugins from environment variables
+        expected_plugins="${{ steps.extract-plugin-id.outputs.plugin-id }}"
+        if [ -n "${{ inputs.other_plugins }}" ]; then
+          expected_plugins="$expected_plugins,${{ inputs.other_plugins }}"
+        fi
+
+        # Convert comma-separated list to array for easier processing
+        IFS=',' read -ra PLUGIN_ARRAY <<< "$expected_plugins"
+
+        echo "Expected plugins: $expected_plugins"
+
+        while true; do
+          current_time=$(date +%s)
+          elapsed=$((current_time - start_time))
+
+          if [ $elapsed -ge $timeout ]; then
+            echo "Timeout reached: Grafana did not become ready within 5 minutes."
+            echo "Container logs:"
+            docker logs ${{ steps.extract-plugin-id.outputs.plugin-id }} --tail 50
+            echo "Installed plugins:"
+            curl -s http://localhost:3000/api/plugins 2>/dev/null | jq -r '.[].id' 2>/dev/null || echo "Could not fetch plugins list"
+            exit 1
+          fi
+
+          # Check if Grafana is responding to health checks
+          if curl -f -s http://localhost:3000/api/health > /dev/null 2>&1; then
+            echo "Grafana health endpoint is responding! (${elapsed}s elapsed)"
+
+            # Check if API is ready
+            if curl -f -s http://localhost:3000/api/org > /dev/null 2>&1; then
+              echo "Grafana API is ready! Now checking plugins..."
+
+              # Get list of installed plugins
+              installed_plugins=$(curl -s http://localhost:3000/api/plugins 2>/dev/null | jq -r '.[].id' 2>/dev/null)
+
+              if [ $? -eq 0 ] && [ -n "$installed_plugins" ]; then
+                # Check if all expected plugins are installed
+                all_plugins_ready=true
+                missing_plugins=""
+
+                for expected_plugin in "${PLUGIN_ARRAY[@]}"; do
+                  # Trim whitespace
+                  expected_plugin=$(echo "$expected_plugin" | xargs)
+                  if [ -n "$expected_plugin" ]; then
+                    if ! echo "$installed_plugins" | grep -q "^${expected_plugin}$"; then
+                      all_plugins_ready=false
+                      missing_plugins="$missing_plugins $expected_plugin"
+                    fi
+                  fi
+                done
+
+                if [ "$all_plugins_ready" = true ]; then
+                  echo "All expected plugins are installed and ready!"
+                  break
+                else
+                  echo "Still waiting for plugins:$missing_plugins (${elapsed}s elapsed)"
+                fi
+              else
+                echo "Could not fetch plugins list, retrying... (${elapsed}s elapsed)"
+              fi
+            else
+              echo "Grafana health OK, but API not fully ready yet... (${elapsed}s elapsed)"
+            fi
+          else
+            echo "Waiting for Grafana health endpoint... (${elapsed}s elapsed)"
+          fi
+
+          sleep 10
+        done
+      working-directory: ${{ inputs.plugin-directory }}
+
+    - name: Run Playwright tests
+      shell: bash
+      env:
+        NODE_ENV: production
+      run: npx playwright test
+      working-directory: ${{ inputs.plugin-directory }}
+
+    - name: Stop grafana docker
+      run: docker compose -f docker-compose.e2e.yaml down
+      working-directory: ${{ inputs.plugin-directory }}
+      shell: bash
+
+    - name: Upload E2E report
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: ${{ inputs.upload_report_path }}
+        path: playwright-report/
+        retention-days: 30
+
+    - name: Upload E2E videos
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: ${{ inputs.upload_videos_path }}
+        path: test-results/
+        retention-days: 30
+
+branding:
+  icon: "shield"
+  color: "green"