Merge pull request #2083 from input-output-hk/ensemble/2027/enhance-nightly-backward-compatibility-testing-workflow

CI: enhance nightly backward compatibility testing workflow

Author: dlachaume

.github/workflows/nightly-backward-compatibility.yml

@@ -1,9 +1,10 @@
 name: Nightly backward compatibility
 
 on:
+  # Important note about scheduled workflows:
+  # Notifications for scheduled workflows are sent to the user who last modified the cron syntax in the workflow file.
   schedule:
-    - cron: "0 2 * * *"
-    - cron: "0 14 * * *"
+    - cron: "30 2 * * *"
   workflow_dispatch:
     inputs:
       total-releases:
@@ -15,22 +16,50 @@ on:
         description: "Cardano node version used in e2e"
         required: true
         type: string
-        default: "10.1.1"
+        default: "10.1.2"
 
 jobs:
+  prepare-env-variables:
+    runs-on: ubuntu-22.04
+    outputs:
+      total_releases: ${{ steps.set-env.outputs.total_releases }}
+      cardano_node_version: ${{ steps.set-env.outputs.cardano_node_version }}
+    steps:
+      - name: Prepare env variables
+        id: set-env
+        shell: bash
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "total_releases=3" >> $GITHUB_OUTPUT
+            echo 'cardano_node_version=["10.1.2"]' >> $GITHUB_OUTPUT
+          else
+            echo "total_releases=${{ inputs.total-releases }}" >> $GITHUB_OUTPUT
+            echo "cardano_node_version=[\"${{ inputs.cardano-node-version }}\"]" >> $GITHUB_OUTPUT
+          fi
+
   prepare-binaries:
     runs-on: ubuntu-22.04
+    needs: [prepare-env-variables]
     outputs:
       tags: ${{ steps.tags-test-lab.outputs.tags }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Download releases artifacts binaries
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
         run: |
-          ./.github/workflows/scripts/download-distribution-binaries.sh ${{ inputs.total-releases }
+          ./.github/workflows/scripts/download-distribution-binaries.sh ${{ needs.prepare-env-variables.outputs.total_releases }}
+
+      - name: Install stable toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
 
       - name: Build e2e
+        shell: bash
         run: |
           cargo build --release --bin mithril-end-to-end
           cp ./target/release/mithril-end-to-end ./mithril-binaries/unstable
@@ -43,21 +72,21 @@ jobs:
 
       - name: Prepare test lab tags
         id: tags-test-lab
+        shell: bash
         run: |
-          TAGS=$(cat ./mithril-binaries/tags.json)
+          TAGS=$(jq -c '.' ./mithril-binaries/tags.json)
           echo "Test Lab Tags: $TAGS"
           echo "tags=$TAGS" >> $GITHUB_OUTPUT
 
   e2e:
     runs-on: ubuntu-22.04
-    needs: [prepare-binaries]
+    needs: [prepare-env-variables, prepare-binaries]
     strategy:
       fail-fast: false
       matrix:
         tag: ${{ fromJSON(needs.prepare-binaries.outputs.tags) }}
-        node:
-          [mithril-aggregator, mithril-client, mithril-signer, mithril-relay]
-        cardano_node_version: ${{ inputs.cardano-node-version }}
+        node: [mithril-aggregator, mithril-client, mithril-signer]
+        cardano_node_version: ${{ fromJSON(needs.prepare-env-variables.outputs.cardano_node_version) }}
         run_id: ["#1"]
 
     steps:
@@ -71,6 +100,7 @@ jobs:
           path: ./mithril-binaries
 
       - name: Prepare binaries
+        shell: bash
         run: |
           mkdir -p mithril-binaries/e2e
           cp ./mithril-binaries/unstable/* ./mithril-binaries/e2e
@@ -84,9 +114,118 @@ jobs:
           mkdir artifacts
 
       - name: Run E2E tests
+        shell: bash
         run: |
-          ./mithril-binaries/e2e/mithril-end-to-end -vvv \\
-                --bin-directory ./mithril-binaries/e2e \\
-                --work-directory=./artifacts \\
-                --devnet-scripts-directory=./mithril-test-lab/mithril-devnet \\
-                --cardano-node-version ${{ matrix.cardano_node_version }}
+          ./mithril-binaries/e2e/mithril-end-to-end -vvv \
+            --bin-directory ./mithril-binaries/e2e \
+            --work-directory=./artifacts \
+            --devnet-scripts-directory=./mithril-test-lab/mithril-devnet \
+            --cardano-node-version ${{ matrix.cardano_node_version }} \
+            --cardano-slot-length 0.25 \
+            --cardano-epoch-length 45.0 \
+          && echo "SUCCESS=true" >> $GITHUB_ENV \
+          || (echo "SUCCESS=false" >> $GITHUB_ENV && exit 1)
+
+      - name: Define the JSON file name for the test result
+        shell: bash
+        if: success() || failure()
+        run: echo "RESULT_FILE_NAME=e2e-test-result-run_${{ github.run_number }}-attempt_${{ github.run_attempt }}-tag_${{ matrix.tag }}-node-${{ matrix.node }}-cardano-${{ matrix.cardano_node_version }}-run_id_${{ matrix.run_id }}" >> $GITHUB_ENV
+
+      - name: Write test result JSON
+        if: success() || failure()
+        shell: bash
+        run: |
+          AGGREGATOR_TAG="unstable"
+          SIGNER_TAG="unstable"
+          CLIENT_TAG="unstable"
+
+          case "$NODE" in
+            mithril-aggregator)
+              AGGREGATOR_TAG="${{ matrix.tag }}"
+              ;;
+            mithril-signer)
+              SIGNER_TAG="${{ matrix.tag }}"
+              ;;
+            mithril-client)
+              CLIENT_TAG="${{ matrix.tag }}"
+              ;;
+          esac
+
+          jq -n --arg TAG "${{ matrix.tag }}" \
+                --arg NODE "${{ matrix.node }}" \
+                --arg CARDANO_NODE "${{ matrix.cardano_node_version }}" \
+                --arg AGGREGATOR "$AGGREGATOR_TAG" \
+                --arg SIGNER "$SIGNER_TAG" \
+                --arg CLIENT "$CLIENT_TAG" \
+                --argjson SUCCESS "${{ env.SUCCESS }}" \
+                '{tag: $TAG, node: $NODE, mithril_signer: $SIGNER, mithril_aggregator: $AGGREGATOR, mithril_client: $CLIENT, cardano_node_version: $CARDANO_NODE, success: $SUCCESS}' \
+                > ./${{ env.RESULT_FILE_NAME }}.json
+
+      - name: Upload test result JSON
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.RESULT_FILE_NAME }}
+          path: ./${{ env.RESULT_FILE_NAME }}.json
+
+      - name: Upload E2E Tests Artifacts
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: mithril-e2e-tests-artifacts-run_${{ github.run_number }}-attempt_${{ github.run_attempt }}-tag_${{ matrix.tag }}-node-${{ matrix.node }}-cardano-${{ matrix.cardano_node_version }}-run_id_${{ matrix.run_id }}
+          path: |
+            ./artifacts/*
+            # including node.sock makes the upload fails so exclude them:
+            !./artifacts/**/node.sock
+            # exclude cardano tools, saving ~50mb of data:
+            !./artifacts/devnet/cardano-cli
+            !./artifacts/devnet/cardano-node
+          if-no-files-found: error
+
+  summarize-test-results:
+    runs-on: ubuntu-22.04
+    needs: [e2e]
+    if: success() || failure()
+
+    steps:
+      - name: Download all test result artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: ./test-results
+          pattern: e2e-test-result*
+          merge-multiple: true
+
+      - name: Concatenate JSON result files into summary.json
+        shell: bash
+        run: |
+          jq -s '.' ./test-results/e2e-test-result-*.json > ./test-results/summary.json
+
+      - name: Add distributions backward compatibility summary
+        shell: bash
+        run: |
+          CHECK_MARK=":heavy_check_mark:"
+          CROSS_MARK=":no_entry:"
+
+          echo "## Distributions backward compatibility" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "This is the compatibility report of previous distributions nodes with the current unstable nodes." >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "| Compatibility | mithril-signer | mithril-aggregator | mithril-client |" >> $GITHUB_STEP_SUMMARY
+          echo "| --- | :---: | :---: | :---: |" >> $GITHUB_STEP_SUMMARY
+
+          # Transform summary.json into Markdown table rows
+          jq -r --arg CHECK_MARK "$CHECK_MARK" --arg CROSS_MARK "$CROSS_MARK" \
+            'group_by(.tag) |
+            sort_by(.[0].tag | tonumber) | reverse |
+            .[] |
+            {
+              tag: .[0].tag, 
+              signer: (map(select(.node == "mithril-signer") | if .success then $CHECK_MARK else $CROSS_MARK end) | join("")),
+              aggregator: (map(select(.node == "mithril-aggregator") | if .success then $CHECK_MARK else $CROSS_MARK end) | join("")),
+              client: (map(select(.node == "mithril-client") | if .success then $CHECK_MARK else $CROSS_MARK end) | join(""))
+            } |
+            "| `\(.tag)` | \(.signer) | \(.aggregator) | \(.client) |"' "./test-results/summary.json" >> $GITHUB_STEP_SUMMARY
+
+          cat "$GITHUB_STEP_SUMMARY"