Release/0.28.0 (#428)

Co-authored-by: mateuszaaa <mateusz@mangata.finance>
Co-authored-by: goncer <gonza.cerro@gmail.com>
Co-authored-by: Misha Kolesnik <tenequm@gmail.com>
Co-authored-by: iStrike7 <40371953+iStrike7@users.noreply.github.com>
Co-authored-by: Michal Danco <devdanco@gmail.com>
Co-authored-by: Mateusz Nowakowski <mat13mn@gmail.com>
Co-authored-by: Marian Vanderka <vanderka.marian@gmail.com>

Author: 7 people

.github/workflows/branch-develop.yml

@@ -82,6 +82,15 @@ jobs:
     with:
       env: fungible
       version: ${{ needs.init.outputs.GLOBAL_VERSION }}
+
+  performance-tests:
+    name: performance tests
+    needs: [init, deploy-fungible]
+    if: |
+      (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == ':rocket: deploy' && contains(github.event.pull_request.labels.*.name, 'full-benchmarks')) ||
+      (github.event_name == 'pull_request' && github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, ':rocket: deploy') && contains(github.event.pull_request.labels.*.name, 'full-benchmarks'))
+    uses: ./.github/workflows/reusable-perfomance-tests.yml
+    secrets: inherit
 
   clean-up-fungible:
     name: Delete fungible environment
@@ -131,4 +140,4 @@ jobs:
       e2eBranch: ${{ github.event.inputs.e2eBranch }}
       parachainDocker: ${{ github.event.inputs.parachainDocker }}
       skipBuild: ${{ github.event.inputs.skipBuild }}
-      globalVersion: ${{ needs.init.outputs.GLOBAL_VERSION }}
+      globalVersion: ${{ needs.init.outputs.GLOBAL_VERSION }}

.github/workflows/deploy-rococo.yml

@@ -0,0 +1,63 @@
+name: Deploy mangata-node to `rococo`
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to be assigned to the built image"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      version:
+        description: "Version to be assigned to the built image"
+        required: true
+        type: string
+
+concurrency: rococo_environment
+
+jobs:
+  deploy_rococo:
+    runs-on: ubuntu-latest
+    environment: rococo
+    env:
+      version_file: helmfiles/rococo/.version
+      GH_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }}
+    steps:
+      - name: Verify specified version exists
+        run: docker pull mangatasolutions/mangata-node:${{ inputs.version }}
+      
+      - name: Checkout `mangata-finance/gitops` repository
+        uses: actions/checkout@v3
+        with:
+          repository:  mangata-finance/gitops
+          ref: main
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+
+      - name: Push updated version to `gitops` repository
+        id: version
+        run: |
+          echo -n '${{ inputs.version }}' > ${{ env.version_file }}
+          git config user.name mangatafinance
+          git config user.email solutions@mangata.finance
+          git add .
+          git commit -m 'chore: deploy ${{ inputs.version }} to rococo environment' || \
+            echo "changed=false" >> $GITHUB_OUTPUT
+          git push
+      
+      - name: Watch deployment workflow run
+        run: |
+          # Rerun previous workflow if version didn't change
+          [ -n '${{ steps.version.outputs.changed }}' ] && gh run rerun $(gh run list -w deploy-rococo.yml -L1 --json databaseId --jq '.[].databaseId')
+          
+          sleep 5 && gh run list -L10 -w deploy-rococo.yml
+          export RUN_ID=$(gh run list -w deploy-rococo.yml -L1 --json databaseId --jq '.[].databaseId')
+          
+          if [[ ($(gh run view "$RUN_ID" --json headSha --jq '.headSha') != $(git rev-parse HEAD)) && -z '${{ steps.version.outputs.changed }}' ]]; then
+            echo "Commited git SHA doesn't match with the SHA of the retrieved Run ID"
+            exit 1
+          fi
+          
+          echo "Deployed version: ${{ inputs.version }}" >> $GITHUB_STEP_SUMMARY
+          echo "Deployment workflow URL: $(gh run view "$RUN_ID" --json url --jq '.url')" >> $GITHUB_STEP_SUMMARY
+          gh run watch "$RUN_ID" --exit-status

.github/workflows/reusable-build-and-test.yml

@@ -13,7 +13,7 @@ on:
         required: true
         type: string
       builder_image:
-        default: mangatasolutions/node-builder:multi-nightly-2022-11-21
+        default: mangatasolutions/node-builder:multi-nightly-2022-11-15
         description: Docker image used for Rust builds
         required: false
         type: string
@@ -51,15 +51,15 @@ jobs:
             /usr/local/cargo/registry
             ~/.cache/sccache
           key: node-cache-${{ inputs.cache_version }}-${{ hashFiles('Cargo.lock') }}
-      
+
       - name: Compile mangata-node code
         run: cargo build --release --no-default-features --features=mangata-rococo,mangata-kusama
 
       - name: Rename wasms
         run: |
           cp target/release/wbuild/mangata-kusama-runtime/mangata_kusama_runtime.compact.compressed.wasm ./mangata_kusama_runtime-${{ inputs.version }}.compact.compressed.wasm
           cp target/release/wbuild/mangata-rococo-runtime/mangata_rococo_runtime.compact.compressed.wasm ./mangata_rococo_runtime-${{ inputs.version }}.compact.compressed.wasm
-      
+
       - name: Build and push Docker image
         run: |
           docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_TOKEN }}
@@ -80,7 +80,7 @@ jobs:
         run: |
           cp target/release/wbuild/mangata-kusama-runtime/mangata_kusama_runtime.compact.compressed.wasm ./mangata_kusama_runtime-${{ inputs.version }}-fast.compact.compressed.wasm
           cp target/release/wbuild/mangata-rococo-runtime/mangata_rococo_runtime.compact.compressed.wasm ./mangata_rococo_runtime-${{ inputs.version }}-fast.compact.compressed.wasm
-      
+
       - name: Build and push Docker image with fast runtime
         run: |
           docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_TOKEN }}
@@ -102,7 +102,7 @@ jobs:
             ./mangata_rococo_runtime-${{ inputs.version }}.compact.compressed.wasm
             ./mangata_kusama_runtime-${{ inputs.version }}-fast.compact.compressed.wasm
             ./mangata_rococo_runtime-${{ inputs.version }}-fast.compact.compressed.wasm
-      
+
       - name: Fix permissions on self-hosted runner
         if: always()
         run: chown -R 1003:1004 $GITHUB_WORKSPACE
@@ -216,11 +216,11 @@ jobs:
           key: cargo-try-runtime-${{ inputs.cache_version }}-${{ hashFiles('Cargo.lock') }}
 
       - name: Run try-runtime Rococo testnet
-        run: cargo run --release --features=try-runtime,mangata-rococo try-runtime --chain=public-testnet --execution=Native on-runtime-upgrade live --uri wss://roccoco-testnet-collator-01.mangatafinance.cloud:443
+        run: cargo run --release --features=try-runtime,mangata-rococo try-runtime --chain=public-testnet --runtime=target/release/wbuild/mangata-rococo-runtime/mangata_rococo_runtime.wasm on-runtime-upgrade live --uri wss://roccoco-testnet-collator-01.mangatafinance.cloud:443
 
       - name: Run try-runtime Kusama Mainnet
-        run: cargo run --release --features=try-runtime try-runtime --chain=kusama-mainnet --execution=Native on-runtime-upgrade live --uri wss://prod-kusama-collator-01.mangatafinance.cloud:443
-      
+        run: cargo run --release --features=try-runtime try-runtime --chain=kusama-mainnet --runtime=target/release/wbuild/mangata-kusama-runtime/mangata_kusama_runtime.wasm on-runtime-upgrade live --uri wss://prod-kusama-collator-01.mangatafinance.cloud:443
+
       - name: Fix permissions on self-hosted runner
         if: always()
         run: chown -R 1003:1004 $GITHUB_WORKSPACE
@@ -274,7 +274,7 @@ jobs:
           --repeat ${{ env.REPEATS }} \
           --template ./templates/module-weight-template.hbs \
           --output ./benchmarks/
-      
+
       - name: Run block & extrinsic overhead benchmarks
         run: |
           target/release/mangata-node benchmark overhead --execution native --chain dev -lblock_builder=debug --max-ext-per-block 50000 --base-path .

.github/workflows/reusable-deploy.yml

@@ -73,6 +73,9 @@ jobs:
       - run: |
           export ENVIRONMENT=${{ env.ENV_REF }}
           export IMAGE_TAG=${{ inputs.version }}
+          
+          # https://dev.to/derlin/helmfile-difference-between-sync-and-apply-helm-3-28o1
+          export HELM_DIFF_THREE_WAY_MERGE=true
           helmfile apply -e ${{ inputs.env }}
         working-directory: devops/helmfiles
       

.github/workflows/reusable-e2e-tests.yml

@@ -1,6 +1,27 @@
 name: Reusable e2e tests workflow
 
 on:
+  workflow_dispatch:
+    inputs:
+      e2eBranch:
+        description: "Name of the e2e target branch"
+        type: string
+        required: false
+        default: "main"
+      parachainDocker:
+        description: "Name of the parachain docker reference"
+        type: string
+        required: false
+        default: "mangatasolutions/mangata-node:latest"
+      skipBuild:
+        description: "Skip build phase"
+        type: string
+        required: false
+        default: false
+      globalVersion:
+        description: "Set Mangata node version."
+        type: string
+        required: true
   workflow_call:
     inputs:
       e2eBranch:
@@ -33,7 +54,19 @@ jobs:
   e2e-test-matrix:
     strategy:
       matrix:
-        command: ["yarn test-parallel --max-workers=32", "yarn test-sequential-no-bootstrap"]
+        include:
+          - command: "yarn test-parallel --max-workers=32"
+            fast: false
+          - command: "yarn test-sequential-no-bootstrap"
+            fast: false
+          - command: "yarn test-seqgasless"
+            fast: false
+          - command: "yarn test-bootstrap"
+            fast: false
+          - command: "yarn test-rewards-bootstrap"
+            fast: false
+          - command: "yarn test-sequential-autocompound"
+            fast: true
     runs-on: [self-hosted, compile]
     timeout-minutes: 180
     env:
@@ -51,6 +84,10 @@ jobs:
       ####IDK, but this is neccesary for reports
       - uses: actions/checkout@v3
 
+      - name: Adapt if fast runtime
+        if:  ${{  !contains(env.PARACHAIN_DOCKER_IMAGE, 'fast') && matrix.fast == true }}
+        run: echo "PARACHAIN_DOCKER_IMAGE=${{ env.PARACHAIN_DOCKER_IMAGE }}-fast" >> $GITHUB_ENV
+      
       - name: Download node Docker image
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-build') && inputs.skipBuild != 'true' }}
         run: docker pull ${{ env.PARACHAIN_DOCKER_IMAGE }}
@@ -84,6 +121,12 @@ jobs:
           repository: mangata-finance/mangata-e2e
           ref: "${{ env.E2EBRANCHNAME }}"
           path: e2eTests
+      
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+          cache: 'yarn'
+          cache-dependency-path: '**/yarn.lock'
 
       - name: Print parachain docker image reference
         run: |
@@ -93,9 +136,13 @@ jobs:
         working-directory: launch
         run: sed -i 's+mangatasolutions/mangata-node:dev+${{ env.PARACHAIN_DOCKER_IMAGE }}+g' config.yml
 
-      - name: Install parachain launch
+      - name: Install parachain launch dependencies
         working-directory: launch
         run: yarn
+      
+      - name: Install e2e tests dependencies
+        working-directory: e2eTests
+        run: yarn
 
       - name: Generate parachain launch config
         working-directory: launch
@@ -109,12 +156,8 @@ jobs:
         working-directory: launch
         run: yarn up
 
-      - name: Install dependencies
-        working-directory: e2eTests
-        run: yarn
-
       - name: Docker ps
-        run: docker ps
+        run: docker ps --no-trunc
 
       - name: Sleep for 2 minutes
         run: sleep 120s #close buffers of videos.

.github/workflows/reusable-perfomance-tests.yml

@@ -0,0 +1,49 @@
+name: Performance tests workflow
+on:
+  workflow_call:
+
+permissions:
+  contents: write
+  id-token: write
+  deployments: write
+  checks: write
+
+jobs:
+  performance-tests:
+    runs-on: [self-hosted, compile]
+    timeout-minutes: 180
+    env:
+      ENV_REF: ${{ format('pr-{0}', github.event.number) }}
+    steps:
+      ####IDK, but this is neccesary for reports
+      - uses: actions/checkout@v3
+
+      - name: Checkout performance-tests
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.DEV_TOOLS_REPO_ACCESS_TOKEN }}
+          repository: mangata-finance/dev-tools
+          path: dev-tools
+
+      - name: Install parachain launch
+        working-directory: dev-tools/fill-block-with-txs
+        run: npm install
+
+      - name: Run performance tests
+        working-directory: dev-tools/fill-block-with-txs
+        run: |
+          ./index.js --url "wss://node-01-ws-${{ env.ENV_REF }}.mangata.online" \
+          --url "wss://node-02-ws-${{ env.ENV_REF }}.mangata.online" \
+          --mnemonic ${{ secrets.E2E_TEST_SUDO_NAME }} \
+          --txs_count 800 \
+          --workers_count 8 \
+          --timeout 300 \
+          --output ../../performance-benchmark.png \
+          --expected_throughput 300 \
+          --number_of_blocks 50
+
+      - name: Upload reports
+        uses: actions/upload-artifact@v3.1.1
+        with:
+          name: performance-benchmark
+          path: ./performance-benchmark.png