Release/v0.30.0 (#501)

Author: majusko

.github/workflows/branch-develop.yml

@@ -83,14 +83,35 @@ jobs:
       env: fungible
       version: ${{ needs.init.outputs.GLOBAL_VERSION }}
 
-  performance-tests:
-    name: performance tests
-    needs: [init, deploy-fungible]
+  run-perf-tests-branch:
+    runs-on: ubuntu-latest
+    needs: [deploy-fungible]
     if: |
       (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == ':rocket: deploy' && contains(github.event.pull_request.labels.*.name, 'full-benchmarks')) ||
       (github.event_name == 'pull_request' && github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, ':rocket: deploy') && contains(github.event.pull_request.labels.*.name, 'full-benchmarks'))
+    steps:
+      - run: echo "Running perf tests in branch"
+  run-perf-tests-dev:
+    runs-on: ubuntu-latest
+    needs: [deploy-dev]
+    steps:
+      - run: echo "Running perf tests in dev"
+      
+  performance-tests:
+    name: performance tests in branch
+    needs: [run-perf-tests-branch]
     uses: ./.github/workflows/reusable-perfomance-tests.yml
     secrets: inherit
+    with:
+      e2eBranch: ${{ github.event.inputs.e2eBranch }}
+  performance-tests-dev:
+    name: performance tests in dev
+    needs: [run-perf-tests-dev]
+    uses: ./.github/workflows/reusable-perfomance-tests.yml
+    secrets: inherit
+    with:
+      e2eBranch: ${{ github.event.inputs.e2eBranch }}
+      targetEnv: "dev"
 
   clean-up-fungible:
     name: Delete fungible environment

.github/workflows/docs.yml

@@ -9,26 +9,37 @@ jobs:
     permissions:
       contents: write
       id-token: write
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, compile]
+    container:
+      image: mangatasolutions/node-builder:multi-nightly-2022-11-21
     steps:
       - uses: actions/checkout@v3
       - uses: google-github-actions/auth@v1
         with:
           workload_identity_provider: '${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}'
           service_account: '${{ secrets.GCP_SERVICE_ACCOUNT }}'
 
-      - name: Pull builder image
-        run: docker pull mangatasolutions/node-builder:0.2
+      - name: Cache the Cargo dependencies
+        uses: mansagroup/gcs-cache-action@v1.0.3
+        with:
+          bucket: mangata-node-ci-cache
+          path: |
+            ${{ github.workspace }}/target
+            /usr/local/cargo/git
+            /usr/local/cargo/registry
+            ~/.cache/sccache
+          key: node-docs-cache-0-${{ hashFiles('Cargo.lock') }}
+      
       - name: Build docs
-        run: |
-          ./docker-cargo.sh fetch || ./docker-cargo.sh fetch || ./docker-cargo.sh fetch || ./docker-cargo.sh fetch || ./docker-cargo.sh fetch
-          ./docker-cargo.sh doc
-        env:
-          DISABLE_TTY: 1
+        run: cargo doc
 
       - name: Deploy to GCP
-        uses: google-github-actions/upload-cloud-storage@v0.10.4
+        uses: google-github-actions/upload-cloud-storage@v1.0.3
         with:
-          path: ./docker-cargo/doc/
+          path: ./target/doc/
           destination: mangata-docs-node
           parent: false
+      
+      - name: Fix permissions on self-hosted runner
+        if: always()
+        run: chown -R 1100:1100 $GITHUB_WORKSPACE

.github/workflows/reusable-build-and-test.yml

@@ -53,7 +53,7 @@ jobs:
           key: node-cache-${{ inputs.cache_version }}-${{ hashFiles('Cargo.lock') }}
 
       - name: Compile mangata-node code
-        run: cargo build --release --no-default-features --features=mangata-rococo,mangata-kusama
+        run: cargo build --locked --release --no-default-features --features=mangata-rococo,mangata-kusama
 
       - name: Rename wasms
         run: |
@@ -105,7 +105,7 @@ jobs:
 
       - name: Fix permissions on self-hosted runner
         if: always()
-        run: chown -R 1004:1005 $GITHUB_WORKSPACE
+        run: chown -R 1100:1100 $GITHUB_WORKSPACE
 
   rustfmt-check:
     name: Formatting check
@@ -166,7 +166,7 @@ jobs:
         run: cargo test -j2
       - name: Fix permissions on self-hosted runner
         if: always()
-        run: chown -R 1004:1005 $GITHUB_WORKSPACE
+        run: chown -R 1100:1100 $GITHUB_WORKSPACE
 
   run-benchmarks-tests:
     name: Run benchmark tests
@@ -190,7 +190,9 @@ jobs:
             /usr/local/cargo/registry
           key: cargo-benchmark-cache-${{ inputs.cache_version }}-${{ hashFiles('Cargo.lock') }}
       - name: Run benchmarks tests
-        run: cargo test --release -j8 --features=runtime-benchmarks -p pallet-xyk -p pallet-issuance -p pallet-multipurpose-liquidity -p pallet-bootstrap -p pallet-fee-lock
+        run: cargo test --release -j8 --features=runtime-benchmarks -p pallet-xyk -p pallet-issuance -p pallet-multipurpose-liquidity -p pallet-fee-lock -p pallet-proof-of-stake -p pallet-proof-of-stake
+      - name: Run benchmarks tests
+        run: cargo test --release -j8 --features=runtime-benchmarks -p pallet-bootstrap
 
   build-and-run-try-runtime:
     name: Run try-runtime checks
@@ -216,14 +218,14 @@ jobs:
           key: cargo-try-runtime-${{ inputs.cache_version }}-${{ hashFiles('Cargo.lock') }}
 
       - name: Run try-runtime Rococo testnet
-        run: cargo run --release --features=try-runtime,mangata-rococo try-runtime --chain=rococo --runtime=target/release/wbuild/mangata-rococo-runtime/mangata_rococo_runtime.wasm on-runtime-upgrade live --uri wss://roccoco-testnet-collator-01.mangatafinance.cloud:443
+        run: cargo run --release --features=try-runtime,mangata-rococo try-runtime --chain=rococo --runtime=target/release/wbuild/mangata-rococo-runtime/mangata_rococo_runtime.wasm on-runtime-upgrade live --uri wss://collator-01-ws-rococo.mangata.online:443
 
       - name: Run try-runtime Kusama Mainnet
         run: cargo run --release --features=try-runtime try-runtime --chain=kusama --runtime=target/release/wbuild/mangata-kusama-runtime/mangata_kusama_runtime.wasm on-runtime-upgrade live --uri wss://prod-kusama-collator-01.mangatafinance.cloud:443
 
       - name: Fix permissions on self-hosted runner
         if: always()
-        run: chown -R 1004:1005 $GITHUB_WORKSPACE
+        run: chown -R 1100:1100 $GITHUB_WORKSPACE
 
   run-benchmarks:
     name: Run runtime benchmarks
@@ -279,14 +281,14 @@ jobs:
         run: |
           target/release/mangata-node benchmark overhead --execution native --chain kusama-local -lblock_builder=debug --max-ext-per-block 50000 --base-path .
           cp block_weights.rs extrinsic_weights.rs ./benchmarks
-
-      - name: Fix permissions on self-hosted runner
-        if: always()
-        run: chown -R 1004:1005 $GITHUB_WORKSPACE
       
       - name: Upload logs and docker images to GitHub
         if: ${{ contains(github.event.pull_request.labels.*.name, 'full-benchmarks') }}
         uses: actions/upload-artifact@v3.1.1
         with:
           name: benchmarks
-          path: ./benchmarks
+          path: ./benchmarks
+
+      - name: Fix permissions on self-hosted runner
+        if: always()
+        run: chown -R 1100:1100 $GITHUB_WORKSPACE

.github/workflows/reusable-e2e-tests.yml

@@ -61,12 +61,21 @@ jobs:
             fast: false
           - command: "yarn test-seqgasless"
             fast: false
+          - command: "yarn test-maintenance"
+            fast: true
           - command: "yarn test-bootstrap"
             fast: false
           - command: "yarn test-rewards-bootstrap"
             fast: false
+          - command: "yarn test-parallel-autocompound"
+            fast: true
           - command: "yarn test-sequential-autocompound"
             fast: true
+          - command: "yarn test-poolliquidity"
+            fast: true
+          - command: "yarn test-governance"
+            fast: true
+
     runs-on: [self-hosted, compile]
     timeout-minutes: 180
     env:
@@ -80,6 +89,7 @@ jobs:
       E2EBRANCHNAME: "main"
       PARACHAIN_DOCKER_IMAGE: ${{ inputs.parachainDocker || format('mangatasolutions/mangata-node:{0}', inputs.globalVersion) }}
     steps:
+     
       - uses: actions/checkout@v3 ####IDK, but this is neccesary for reports
       - name: Adapt if fast runtime
         if:  ${{  !contains(env.PARACHAIN_DOCKER_IMAGE, 'fast') && matrix.fast == true }}
@@ -122,7 +132,7 @@ jobs:
           repository: mangata-finance/mangata-e2e
           ref: "${{ env.E2EBRANCHNAME }}"
           path: e2eTests
-      
+
       - uses: actions/setup-node@v3
         with:
           node-version: '16'
@@ -169,6 +179,7 @@ jobs:
 
       - name: Test Report
         uses: dorny/test-reporter@v1.6.0
+        continue-on-error: true
         if: success() || failure() # run this step even if previous step failed
         with:
           name: E2E report ${{ matrix.command }} # Name of the check run which will be created
@@ -207,3 +218,7 @@ jobs:
         if: always()
         working-directory: launch
         run: yarn down
+
+      - name: Fix permissions on self-hosted runner
+        if: always()
+        run: chown -R 1100:1100 $GITHUB_WORKSPACE

.github/workflows/reusable-perfomance-tests.yml

@@ -1,6 +1,16 @@
 name: Performance tests workflow
 on:
   workflow_call:
+    inputs:
+      e2eBranch:
+        description: "Name of the e2e target branch"
+        type: string
+        required: false
+        default: "main"
+      targetEnv:
+        description: "env name"
+        type: string
+        required: false
 
 permissions:
   contents: write
@@ -13,37 +23,72 @@ jobs:
     runs-on: [self-hosted, compile]
     timeout-minutes: 180
     env:
-      ENV_REF: ${{ format('pr-{0}', github.event.number) }}
+      ENV_REF: ${{ inputs.targetEnv || format('pr-{0}', github.event.number) }}
+      E2EBRANCHNAME: "main"
     steps:
       ####IDK, but this is neccesary for reports
       - uses: actions/checkout@v3
 
+      - name: E2E- Get branch name
+        id: branch-name
+        uses: tj-actions/branch-names@v4.9
+
+      - name: Set Configuration - Develop
+        if: "${{ inputs.targetEnv == 'dev' }}"
+        run: |
+          echo 'TEST_SUDO_NAME=${{ secrets.DEV_SUDO_NAME }}' >> $GITHUB_ENV
+
+      - name: E2E- Calculate if run e2e feature branch or main.
+        run: |
+          echo DEFAULT: E2E test will run with: $E2EBRANCHNAME
+          echo "Running on: ${{ steps.branch-name.outputs.current_branch }}"
+          if [ -n "$(git ls-remote --heads https://github.com/mangata-finance/mangata-e2e.git ${{ steps.branch-name.outputs.current_branch }} --force --quiet)" ]; then echo "E2EBRANCHNAME=${{ steps.branch-name.outputs.current_branch }}" >> $GITHUB_ENV; echo "MATCH - OK" ; elif [ -n "$(git ls-remote --heads https://github.com/mangata-finance/mangata-e2e.git ${{ github.base_ref }} --force --quiet)" ]; then echo "E2EBRANCHNAME=${{ github.base_ref }}" >> $GITHUB_ENV; echo "MATCH - OK" ;  fi
+
+      - name: Decide if main - branch or parameter
+        # if we have something in e2eBranch - override E2EBranchName, else -> E2EBRANCHNAME , that
+        # by default will be main.
+        run: echo "E2EBRANCHNAME=${{ inputs.e2eBranch || env.E2EBRANCHNAME }}" >> $GITHUB_ENV
+
       - name: Checkout performance-tests
         uses: actions/checkout@v3
         with:
-          token: ${{ secrets.DEV_TOOLS_REPO_ACCESS_TOKEN }}
-          repository: mangata-finance/dev-tools
-          path: dev-tools
+          repository: mangata-finance/mangata-e2e
+          ref: "${{ env.E2EBRANCHNAME }}"
+          path: e2e
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "16"
+          cache: "yarn"
+          cache-dependency-path: "**/yarn.lock"
 
-      - name: Install parachain launch
-        working-directory: dev-tools/fill-block-with-txs
-        run: npm install
+      - name: Install e2e tests dependencies
+        working-directory: e2e
+        run: yarn
 
       - name: Run performance tests
-        working-directory: dev-tools/fill-block-with-txs
+        working-directory: e2e/performance
         run: |
-          ./index.js --url "wss://node-01-ws-${{ env.ENV_REF }}.mangata.online" \
-          --url "wss://node-02-ws-${{ env.ENV_REF }}.mangata.online" \
-          --mnemonic ${{ secrets.E2E_TEST_SUDO_NAME }} \
-          --txs_count 800 \
-          --workers_count 8 \
-          --timeout 300 \
-          --output ../../performance-benchmark.png \
-          --expected_throughput 300 \
-          --number_of_blocks 50
+          npx ts-node index.ts \
+          transfer \
+          nodes="wss://node-01-ws-${{ env.ENV_REF }}.mangata.online" \
+          testCaseName=ConcurrentTest \
+          pending=2000 \
+          threadNumber=12 \
+          duration=50 \
+          throughput=200
 
       - name: Upload reports
+        if: always()
         uses: actions/upload-artifact@v3.1.1
         with:
-          name: performance-benchmark
-          path: ./performance-benchmark.png
+          name: performance-benchmarks-report
+          path: |
+            ./e2e/performance/report.html
+            ./e2e/performance/enqueued.txt
+            ./e2e/performance/executed.txt
+            ./e2e/performance/pending.txt
+
+      - name: Fix permissions on self-hosted runner
+        if: always()
+        run: chown -R 1100:1100 $GITHUB_WORKSPACE