Heavily simplify ci matrix build to reduce cache usage

Author: Alenar

.github/workflows/ci.yml

@@ -13,58 +13,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        project: [
-          mithril-core,
-          mithril-common,
-          mithril-aggregator,
-          mithril-client,
-          mithril-signer,
-          mithril-test-lab/mithril-end-to-end,
-          demo/protocol-demo
-        ]
-        os: [ubuntu-22.04]
+        os: [ ubuntu-22.04, macos-12, windows-latest ]
 
         include:
-        - project: mithril-core
-          cargo_project_name: mithril
-          artifacts_pattern: libmithril
-        - project: mithril-common
-          artifacts_pattern: libmithril_common
-        - project: mithril-client
-          os: macos-latest
-        - project: mithril-client
-          os: windows-latest
-        - project: mithril-test-lab/mithril-end-to-end
-          cargo_project_name: mithril-end-to-end
-          artifacts_base_name: mithril-end-to-end
-          artifacts_pattern: mithril-end-to-end
-        - project: demo/protocol-demo
-          cargo_project_name: mithrildemo
-          artifacts_base_name: mithrildemo
-          artifacts_pattern: mithrildemo
-    env:
-      CARGO_PROJECT_NAME: ${{ matrix.project }}
-      ARTIFACTS_BASE_NAME: ${{ matrix.project }}
-      ARTIFACTS_PATTERN: ${{ matrix.project }}
-      
+        # Only build client on windows & mac
+        - os: macos-12
+          build-args: -p mithril-client
+        - os: windows-latest
+          build-args: -p mithril-client 
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Overriding default $CARGO_PROJECT_NAME with matrix value
-        if: ${{ matrix.cargo_project_name }}
-        run: echo "CARGO_PROJECT_NAME=${{ matrix.cargo_project_name }}" >> $GITHUB_ENV
-      
-      - name: Overriding default $ARTIFACTS_BASE_NAME with matrix value
-        if: ${{ matrix.artifacts_base_name }}
-        run: echo "ARTIFACTS_BASE_NAME=${{ matrix.artifacts_base_name }}" >> $GITHUB_ENV
-
-      - name: Overriding default $ARTIFACTS_PATTERN with matrix value
-        if: ${{ matrix.artifacts_pattern }}
-        run: echo "ARTIFACTS_PATTERN=${{ matrix.artifacts_pattern }}" >> $GITHUB_ENV
-
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -81,64 +43,64 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --release --features portable -p ${{ env.CARGO_PROJECT_NAME }}
+          args: --release ${{ matrix.build-args }}
 
-      - name: Publish ${{ matrix.project }} (${{ runner.os }}-${{ runner.arch }})
+      - name: Publish Mithril Distribution (${{ runner.os }}-${{ runner.arch }})
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ env.ARTIFACTS_BASE_NAME }}-${{ runner.os }}-${{ runner.arch }}
+          name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
           path: |
-            target/release/${{ env.ARTIFACTS_PATTERN }}*
+            target/release/libmithril.*
+            target/release/libmithril_common.*
+            target/release/mithril-aggregator
+            target/release/mithril-aggregator.exe
+            target/release/mithril-client
+            target/release/mithril-client.exe
+            target/release/mithril-signer
+            target/release/mithril-signer.exe
+            target/release/mithrildemo
+            target/release/mithrildemo.exe
+          if-no-files-found: error
+
+      - name: Publish End-to-end runner (${{ runner.os }}-${{ runner.arch }})
+        uses: actions/upload-artifact@v3
+        if: ${{ runner.os == 'Linux' }}
+        with:
+          name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }}
+          path: target/release/mithril-end-to-end
           if-no-files-found: error
 
   test:
     strategy:
       fail-fast: false
       matrix:
-        project: [
-          mithril-core,
-          mithril-common,
-          mithril-aggregator,
-          mithril-client,
-          mithril-signer,
-          mithril-test-lab/mithril-end-to-end,
-          demo/protocol-demo
-        ]
-        os: [ubuntu-22.04]
+        title: [ "mithril" ]
+        os: [ ubuntu-22.04, macos-12, windows-latest ]
 
         include:
-          - project: mithril-core
-            cargo_project_name: mithril
-            cargo-args: --release # Mithril core tests are ~2x quicker on release build
-          - project: mithril-common
-          - project: mithril-client
-            os: macos-latest
-          - project: mithril-client
-            os: windows-latest
-          - project: mithril-test-lab/mithril-end-to-end
-            cargo_project_name: mithril-end-to-end
-            artifacts_base_name: mithril-end-to-end
-          - project: demo/protocol-demo
-            cargo_project_name: mithrildemo
-            artifacts_base_name: mithrildemo
-    env:
-      CARGO_PROJECT_NAME: ${{ matrix.project }}
-      ARTIFACTS_BASE_NAME: ${{ matrix.project }}
+          - os: ubuntu-22.04
+            build-args: --workspace
+            test-args: --workspace --exclude mithril # mithril core is tested separately since it's slow
+          - title: mithril-core
+            os: ubuntu-22.04
+            # Mithril core tests are ~2x faster on release build
+            build-args: --release -p mithril
+            test-args: --release -p mithril
+            artifact-suffix: -mithril-core 
+          # Only test client on windows & mac (since its the only binaries supported for those os for now)
+          - os: macos-12
+            build-args: -p mithril-client
+            test-args: -p mithril-client
+          - os: windows-latest
+            build-args: -p mithril-client
+            test-args: -p mithril-client
 
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Overriding default $CARGO_PROJECT_NAME with matrix value
-        if: ${{ matrix.cargo_project_name }}
-        run: echo "CARGO_PROJECT_NAME=${{ matrix.cargo_project_name }}" >> $GITHUB_ENV
-
-      - name: Overriding default $ARTIFACTS_BASE_NAME with matrix value
-        if: ${{ matrix.artifacts_base_name }}
-        run: echo "ARTIFACTS_BASE_NAME=${{ matrix.artifacts_base_name }}" >> $GITHUB_ENV
-
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -152,30 +114,23 @@ jobs:
           key: ${{ runner.os }}-cache-v${{ secrets.CACHE_VERSION }}
 
       - name: Install cargo tools
-        if: ${{ steps.cargo-cache.outputs.cache-hit == false }}
         shell: bash
         run: |
           cargo install cargo2junit 2>/dev/null || true # Suppress the "binary `xyz` already exists in destination" error
 
-      - name: Cargo build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --tests --features portable -p ${{ env.CARGO_PROJECT_NAME }} ${{ matrix.cargo-args }}
-
       - name: Run tests
         shell: bash
         run: |
           set -o pipefail && \
-          cargo test --features portable -p $CARGO_PROJECT_NAME --no-fail-fast ${{ matrix.cargo-args }} \
+          cargo test --features portable --no-fail-fast ${{ matrix.test-args }} \
               -- -Z unstable-options --format json --report-time \
-              | tee >(cargo2junit > test-results-${{ env.artifacts_base_name }}-${{ runner.os }}-${{ runner.arch }}.xml)
+              | tee >(cargo2junit > test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}.xml)
 
       - name: Upload Tests Results
         uses: actions/upload-artifact@v3
         if: ${{ success() || failure() }}
         with:
-          name: test-results-${{ env.ARTIFACTS_BASE_NAME }}-${{ runner.os }}-${{ runner.arch }}
+          name: test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}
           path: |
             ./**/test-results-*.xml
   
@@ -234,22 +189,10 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Download aggregator
-        uses: actions/download-artifact@v3
-        with:
-          name: mithril-aggregator-${{ runner.os }}-${{ runner.arch }}
-          path: ./bin
-
-      - name: Download signer
+      - name: Download binaries
         uses: actions/download-artifact@v3
         with:
-          name: mithril-signer-${{ runner.os }}-${{ runner.arch }}
-          path: ./bin
-
-      - name: Download client
-        uses: actions/download-artifact@v3
-        with:
-          name: mithril-client-${{ runner.os }}-${{ runner.arch }}
+          name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
           path: ./bin
 
       - name: Download rust test runner
@@ -288,60 +231,30 @@ jobs:
     needs: 
       - test
     steps:
-      - name: Download mithril-core Tests Results
+      - name: Download Tests Results (${{ runner.os }}-${{ runner.arch }})
         if: ${{ success() || failure() }}
         uses: actions/download-artifact@v3
         with:
-          name: test-results-mithril-core-${{ runner.os }}-${{ runner.arch }}
+          name: test-results-${{ runner.os }}-${{ runner.arch }}
 
-      - name: Download mithril-common Tests Results
+      - name: Download mithril-core Tests Results (${{ runner.os }}-${{ runner.arch }})
         if: ${{ success() || failure() }}
         uses: actions/download-artifact@v3
         with:
-          name: test-results-mithril-common-${{ runner.os }}-${{ runner.arch }}
-      
-      - name: Download mithril-aggregator Tests Results
-        if: ${{ success() || failure() }}
-        uses: actions/download-artifact@v3
-        with:
-          name: test-results-mithril-aggregator-${{ runner.os }}-${{ runner.arch }}
-
-      - name: Download mithril-client (${{ runner.os }}-${{ runner.arch }}) Tests Results
-        if: ${{ success() || failure() }}
-        uses: actions/download-artifact@v3
-        with:
-          name: test-results-mithril-client-${{ runner.os }}-${{ runner.arch }}
-
-      - name: Download mithril-client (macOS-X64) Tests Results
-        if: ${{ success() || failure() }}
-        uses: actions/download-artifact@v3
-        with:
-          name: test-results-mithril-client-macOS-X64
-
-      - name: Download mithril-client (Windows-X64) Tests Results
-        if: ${{ success() || failure() }}
-        uses: actions/download-artifact@v3
-        with:
-          name: test-results-mithril-client-Windows-X64
+          name: test-results-mithril-core-${{ runner.os }}-${{ runner.arch }}
 
-      - name: Download mithril-signer Tests Results
+      - name: Download Tests Results (macOS-X64)
         if: ${{ success() || failure() }}
         uses: actions/download-artifact@v3
         with:
-          name: test-results-mithril-signer-${{ runner.os }}-${{ runner.arch }}
+          name: test-results-macOS-X64
 
-      - name: Download mithril-end-to-end Tests Results
+      - name: Download Tests Results (Windows-X64)
         if: ${{ success() || failure() }}
         uses: actions/download-artifact@v3
         with:
-          name: test-results-mithril-end-to-end-${{ runner.os }}-${{ runner.arch }}
+          name: test-results-Windows-X64
 
-      - name: Download mithril-demo Tests Results
-        if: ${{ success() || failure() }}
-        uses: actions/download-artifact@v3
-        with:
-          name: test-results-mithrildemo-${{ runner.os }}-${{ runner.arch }}
-      
       - name: Publish Unit Test Results
         if: ${{ success() || failure() }}
         uses: EnricoMi/publish-unit-test-result-action@v1
@@ -350,7 +263,6 @@ jobs:
 
   docker-mithril:
     runs-on: ubuntu-22.04
-    if: ${{ github.event_name == 'push' }}
     needs:
       - build
       - check
@@ -393,10 +305,10 @@ jobs:
             unstable
             type=raw,value={{branch}}-{{sha}}
 
-      - name: Download ${{ matrix.project }} executable
+      - name: Download built artifacts
         uses: actions/download-artifact@v3
         with:
-          name: ${{ matrix.project }}-${{ runner.os }}-${{ runner.arch }}
+          name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
           path: ${{ matrix.project }}
 
       - name: Build and push Docker image
@@ -408,51 +320,33 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
 
   unstable-release:
-    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
     runs-on: ubuntu-22.04
     needs:
       - build
       - test
       - run-test-lab
       - check
     steps:
-      - name: Download mithril-core lib
-        uses: actions/download-artifact@v3
-        with:
-          name: mithril-core-${{ runner.os }}-${{ runner.arch }}
-          path: ./build
-
-      - name: Download aggregator
-        uses: actions/download-artifact@v3
-        with:
-          name: mithril-aggregator-${{ runner.os }}-${{ runner.arch }}
-          path: ./build
-
-      - name: Download signer
-        uses: actions/download-artifact@v3
-        with:
-          name: mithril-signer-${{ runner.os }}-${{ runner.arch }}
-          path: ./build
-
-      - name: Download client (${{ runner.os }}-${{ runner.arch }})
+      - name: Download built artifacts (${{ runner.os }}-${{ runner.arch }})
         uses: actions/download-artifact@v3
         with:
-          name: mithril-client-${{ runner.os }}-${{ runner.arch }}
+          name: mithril-distribution-${{ runner.os }}-${{ runner.arch }}
           path: ./build
 
-      - name: Download client (macOS-X64)
+      - name: Download built artifacts (macOS-X64)
         uses: actions/download-artifact@v3
         with:
-          name: mithril-client-macOS-X64
+          name: mithril-distribution-macOS-X64
           path: ./build
 
-      - name: Download client (Windows-X64)
+      - name: Download built artifacts (Windows-X64)
         uses: actions/download-artifact@v3
         with:
-          name: mithril-client-Windows-X64
+          name: mithril-distribution-Windows-X64
           path: ./build
 
       - name: Update unstable release
+        if: github.ref == 'refs/heads/main'
         uses: marvinpinto/action-automatic-releases@latest
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
@@ -501,6 +395,6 @@ jobs:
         terraform plan -var "image_id=${{ env.BRANCH_NAME }}-${{ steps.slug.outputs.sha8 }}" -var 'private_key=${{ env.GCLOUD_PRIVATE_KEY }}' -var 'google_application_credentials_json=${{ env.GOOGLE_CREDENTIALS }}' -var 'genesis_secret_key=${{ env.GENESIS_SECRET_KEY }}'
 
     - name: Terraform Apply
-      if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+      if: github.ref == 'refs/heads/main'
       run: |
         terraform apply -auto-approve -var "image_id=${{ env.BRANCH_NAME }}-${{ steps.slug.outputs.sha8 }}" -var 'private_key=${{ env.GCLOUD_PRIVATE_KEY }}' -var 'google_application_credentials_json=${{ env.GOOGLE_CREDENTIALS }}' -var 'genesis_secret_key=${{ env.GENESIS_SECRET_KEY }}'