Merge branch 'bridge-next-gen' into transact-from-eth-to-sub

Author: yrong

.github/workflows/parachain.yml

@@ -24,7 +24,7 @@ jobs:
       RUST_BACKTRACE: 1
       RUSTFLAGS: -C debuginfo=1
       SKIP_WASM_BUILD: 1
-      RUST_NIGHTLY_VERSION: nightly-2023-12-28
+      RUST_NIGHTLY_VERSION: nightly-2024-02-08
     steps:
       - uses: actions/checkout@v2
         with:

.github/workflows/release-30_publish_release_draft.yml

@@ -0,0 +1,155 @@
+name: Release - Publish draft
+
+on:
+  push:
+    tags:
+      # Catches v1.2.3 and v1.2.3-rc1
+      - v[0-9]+.[0-9]+.[0-9]+*
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Current release/rc version
+
+jobs:
+  get-rust-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      rustc-stable: ${{ steps.get-rust-versions.outputs.stable }}
+    steps:
+      - id: get-rust-versions
+        run: |
+          RUST_STABLE_VERSION=$(curl -sS https://raw.githubusercontent.com/paritytech/scripts/master/dockerfiles/ci-unified/Dockerfile | grep -oP 'ARG RUST_STABLE_VERSION=\K[^ ]+')
+          echo "stable=$RUST_STABLE_VERSION" >> $GITHUB_OUTPUT
+
+  build-runtimes:
+    uses: "./.github/workflows/srtool.yml"
+    with:
+       excluded_runtimes: "substrate-test bp cumulus-test kitchensink minimal-template parachain-template penpal polkadot-test seedling shell frame-try sp solochain-template"
+
+  publish-release-draft:
+    runs-on: ubuntu-latest
+    needs: [get-rust-versions, build-runtimes]
+    outputs:
+      release_url: ${{ steps.create-release.outputs.html_url }}
+      asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      - name: Prepare tooling
+        run: |
+          URL=https://github.com/chevdor/tera-cli/releases/download/v0.2.4/tera-cli_linux_amd64.deb
+          wget $URL -O tera.deb
+          sudo dpkg -i tera.deb
+          tera --version
+
+      - name: Download artifacts
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+
+      - name: Prepare draft
+        id: draft
+        env:
+          RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ASSET_HUB_ROCOCO_DIGEST: ${{ github.workspace}}/asset-hub-rococo-runtime/asset-hub-rococo-srtool-digest.json
+          ASSET_HUB_WESTEND_DIGEST: ${{ github.workspace}}/asset-hub-westend-runtime/asset-hub-westend-srtool-digest.json
+          BRIDGE_HUB_ROCOCO_DIGEST: ${{ github.workspace}}/bridge-hub-rococo-runtime/bridge-hub-rococo-srtool-digest.json
+          BRIDGE_HUB_WESTEND_DIGEST: ${{ github.workspace}}/bridge-hub-westend-runtime/bridge-hub-westend-srtool-digest.json
+          COLLECTIVES_WESTEND_DIGEST: ${{ github.workspace}}/collectives-westend-runtime/collectives-westend-srtool-digest.json
+          CONTRACTS_ROCOCO_DIGEST: ${{ github.workspace}}/contracts-rococo-runtime/contracts-rococo-srtool-digest.json
+          CORETIME_ROCOCO_DIGEST: ${{ github.workspace}}/coretime-rococo-runtime/coretime-rococo-srtool-digest.json
+          CORETIME_WESTEND_DIGEST: ${{ github.workspace}}/coretime-westend-runtime/coretime-westend-srtool-digest.json
+          GLUTTON_WESTEND_DIGEST: ${{ github.workspace}}/glutton-westend-runtime/glutton-westend-srtool-digest.json
+          PEOPLE_ROCOCO_DIGEST: ${{ github.workspace}}/people-rococo-runtime/people-rococo-srtool-digest.json
+          PEOPLE_WESTEND_DIGEST: ${{ github.workspace}}/people-westend-runtime/people-westend-srtool-digest.json
+          ROCOCO_DIGEST: ${{ github.workspace}}/rococo-runtime/rococo-srtool-digest.json
+          WESTEND_DIGEST: ${{ github.workspace}}/westend-runtime/westend-srtool-digest.json
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          export REF1=$(get_latest_release_tag)
+          if [[ -z "${{ inputs.version }}" ]]; then
+            export REF2="${{ github.ref }}"
+          else
+            export REF2="${{ inputs.version }}"
+          fi
+          echo "REL_TAG=$REF2" >> $GITHUB_ENV
+          export VERSION=$(echo "$REF2" | sed -E 's/^v([0-9]+\.[0-9]+\.[0-9]+).*$/\1/')
+
+          ./scripts/release/build-changelogs.sh
+
+          echo "Checking the folder state"
+          pwd
+          ls -la scripts/release
+
+      - name: Archive artifact context.json
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: release-notes-context
+          path: |
+            scripts/release/context.json
+            **/*-srtool-digest.json
+
+      - name: Create draft release
+        id: create-release
+        uses: actions/create-release@0cb9c9b65d5d1901c1f53e5e66eaf4afd303e70e # v1.1.4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.REL_TAG }}
+          release_name: Polkadot ${{ env.REL_TAG }}
+          body_path: ${{ github.workspace}}/scripts/release/RELEASE_DRAFT.md
+          draft: true
+
+  publish-runtimes:
+    needs: [ build-runtimes, publish-release-draft ]
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJSON(needs.build-runtimes.outputs.published_runtimes) }}
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      - name: Download artifacts
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+
+      - name: Get runtime info
+        env:
+          JSON: release-notes-context/${{ matrix.chain }}-runtime/${{ matrix.chain }}-srtool-digest.json
+        run: |
+          >>$GITHUB_ENV echo ASSET=$(find ${{ matrix.chain }}-runtime -name '*.compact.compressed.wasm')
+          >>$GITHUB_ENV echo SPEC=$(<${JSON} jq -r .runtimes.compact.subwasm.core_version.specVersion)
+
+      - name: Upload compressed ${{ matrix.chain }} v${{ env.SPEC }} wasm
+        if: ${{ matrix.chain  != 'rococo-parachain' }}
+        uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 #v1.0.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.publish-release-draft.outputs.asset_upload_url }}
+          asset_path: ${{ env.ASSET }}
+          asset_name: ${{ matrix.chain }}_runtime-v${{ env.SPEC }}.compact.compressed.wasm
+          asset_content_type: application/wasm
+
+  post_to_matrix:
+    runs-on: ubuntu-latest
+    needs: publish-release-draft
+    strategy:
+      matrix:
+        channel:
+          - name: "Team: RelEng Internal"
+            room: '!GvAyzgCDgaVrvibaAF:parity.io'
+
+    steps:
+      - name: Send Matrix message to ${{ matrix.channel.name }}
+        uses: s3krit/matrix-message-action@70ad3fb812ee0e45ff8999d6af11cafad11a6ecf # v0.0.3
+        with:
+          room_id: ${{ matrix.channel.room }}
+          access_token: ${{ secrets.RELEASENOTES_MATRIX_V2_ACCESS_TOKEN }}
+          server: m.parity.io
+          message: |
+            **New version of polkadot tagged**: ${{ github.ref }}<br/>
+            Draft release created: ${{ needs.publish-release-draft.outputs.release_url }}

.gitlab-ci.yml

@@ -119,19 +119,26 @@ default:
 #
 .forklift-cache:
   before_script:
-    - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift  -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"'
-    - chmod +x forklift
     - mkdir ~/.forklift
     - cp $FL_FORKLIFT_CONFIG ~/.forklift/config.toml
-    - shopt -s expand_aliases
-    - export PATH=$PATH:$(pwd)
-    - |
+    - >
       if [ "$FORKLIFT_BYPASS" != "true" ]; then
-      echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"
-      alias cargo="forklift cargo"
+        echo "FORKLIFT_BYPASS not set"; 
+        if command -v forklift >/dev/null 2>&1; then
+          echo "forklift already exists"; 
+          forklift version
+        else
+          echo "forklift does not exist, downloading";
+          curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift  -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"; 
+          chmod +x forklift;
+          export PATH=$PATH:$(pwd);
+          echo ${FL_FORKLIFT_VERSION};
+        fi
+        echo "Creating alias cargo='forklift cargo'"; 
+        shopt -s expand_aliases;
+        alias cargo="forklift cargo";
       fi
     #
-    - echo "FL_FORKLIFT_VERSION ${FL_FORKLIFT_VERSION}"
 
 .common-refs:
   rules:

.gitlab/pipeline/build.yml

@@ -91,40 +91,39 @@ build-rustdoc:
     - .run-immediately
   variables:
     SKIP_WASM_BUILD: 1
-    RUSTDOCFLAGS: "--default-theme=ayu --html-in-header ./docs/sdk/headers/header.html --extend-css ./docs/sdk/headers/theme.css"
+    RUSTDOCFLAGS: "-Dwarnings --default-theme=ayu --html-in-header ./docs/sdk/assets/header.html --extend-css ./docs/sdk/assets/theme.css --html-after-content ./docs/sdk/assets/after-content.html"
   artifacts:
     name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
     when: on_success
     expire_in: 1 days
     paths:
       - ./crate-docs/
   script:
-    # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
-    - time cargo doc --features try-runtime,experimental --workspace --no-deps
+    - time cargo doc --all-features --workspace --no-deps
     - rm -f ./target/doc/.lock
     - mv ./target/doc ./crate-docs
     # Inject Simple Analytics (https://www.simpleanalytics.com/) privacy preserving tracker into
     # all .html files
-    - |
+    - >
       inject_simple_analytics() {
-        local path="$1"
-        local script_content="<script async defer src=\"https://apisa.parity.io/latest.js\"></script><noscript><img src=\"https://apisa.parity.io/latest.js\" alt=\"\" referrerpolicy=\"no-referrer-when-downgrade\" /></noscript>"
+        local path="$1";
+        local script_content="<script async defer src=\"https://apisa.parity.io/latest.js\"></script><noscript><img src=\"https://apisa.parity.io/latest.js\" alt=\"\" referrerpolicy=\"no-referrer-when-downgrade\" /></noscript>";
 
         # Function that inject script into the head of an html file using sed.
         process_file() {
-            local file="$1"
-            echo "Adding Simple Analytics script to $file"
-            sed -i "s|</head>|$script_content</head>|" "$file"
-        }
-        export -f process_file
-        # xargs runs process_file in seperate shells without access to outer variables.
-        # to make script_content available inside process_file, export it as an env var here.
-        export script_content
+            local file="$1";
+            echo "Adding Simple Analytics script to $file";
+            sed -i "s|</head>|$script_content</head>|" "$file";
+        };
+        export -f process_file;
+        # xargs runs process_file in separate shells without access to outer variables.
+        # make script_content available inside process_file, export it as an env var here.
+        export script_content;
 
         # Modify .html files in parallel using xargs, otherwise it can take a long time.
-        find "$path" -name '*.html' | xargs -I {} -P "$(nproc)" bash -c 'process_file "$@"' _ {}
-      }
-      inject_simple_analytics "./crate-docs"
+        find "$path" -name '*.html' | xargs -I {} -P "$(nproc)" bash -c 'process_file "$@"' _ {};
+      };
+      inject_simple_analytics "./crate-docs";
     - echo "<meta http-equiv=refresh content=0;url=polkadot_sdk_docs/index.html>" > ./crate-docs/index.html
 
 build-implementers-guide:

.gitlab/pipeline/check.yml

@@ -7,8 +7,8 @@ cargo-clippy:
   variables:
     RUSTFLAGS: "-D warnings"
   script:
-    - SKIP_WASM_BUILD=1 cargo clippy --all-targets --locked --workspace
-    - SKIP_WASM_BUILD=1 cargo clippy --all-targets --all-features --locked --workspace
+    - SKIP_WASM_BUILD=1 cargo clippy --all-targets --locked --workspace --quiet
+    - SKIP_WASM_BUILD=1 cargo clippy --all-targets --all-features --locked --workspace --quiet
 
 check-try-runtime:
   stage: check
@@ -104,23 +104,20 @@ check-toml-format:
     - .docker-env
     - .test-pr-refs
   script:
-    - |
-      export RUST_LOG=remote-ext=debug,runtime=debug
-
-      echo "---------- Downloading try-runtime CLI ----------"
-      curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.4/try-runtime-x86_64-unknown-linux-musl -o try-runtime
-      chmod +x ./try-runtime
-      echo "Using try-runtime-cli version:"
-      ./try-runtime --version
-
-      echo "---------- Building ${PACKAGE} runtime ----------"
-      time cargo build --release --locked -p "$PACKAGE" --features try-runtime
-
-      echo "---------- Executing on-runtime-upgrade for ${NETWORK} ----------"
+    - export RUST_LOG=remote-ext=debug,runtime=debug
+    - echo "---------- Downloading try-runtime CLI ----------"
+    - curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.4/try-runtime-x86_64-unknown-linux-musl -o try-runtime
+    - chmod +x ./try-runtime
+    - echo "Using try-runtime-cli version:"
+    - ./try-runtime --version
+    - echo "---------- Building ${PACKAGE} runtime ----------"
+    - time cargo build --release --locked -p "$PACKAGE" --features try-runtime
+    - echo "---------- Executing on-runtime-upgrade for ${NETWORK} ----------"
+    - >
       time ./try-runtime ${COMMAND_EXTRA_ARGS} \
-          --runtime ./target/release/wbuild/"$PACKAGE"/"$WASM" \
-          on-runtime-upgrade --disable-spec-version-check --checks=all ${SUBCOMMAND_EXTRA_ARGS} live --uri ${URI}
-      sleep 5
+        --runtime ./target/release/wbuild/"$PACKAGE"/"$WASM" \
+        on-runtime-upgrade --disable-spec-version-check --checks=all ${SUBCOMMAND_EXTRA_ARGS} live --uri ${URI}
+    - sleep 5
 
 # Check runtime migrations for Parity managed relay chains
 check-runtime-migration-westend:

.gitlab/pipeline/publish.yml

@@ -113,17 +113,17 @@ trigger_workflow:
       artifacts: true
   script:
     - echo "Triggering workflow"
-    - |
+    - >
       for benchmark in $(ls charts/*.json); do
-        export bencmark_name=$(basename $benchmark)
-        echo "Benchmark: $bencmark_name"
-        export benchmark_dir=$(echo $bencmark_name | sed 's/\.json//')
+        export benchmark_name=$(basename $benchmark);
+        echo "Benchmark: $benchmark_name";
+        export benchmark_dir=$(echo $benchmark_name | sed 's/\.json//');
         curl -q -X POST \
               -H "Accept: application/vnd.github.v3+json" \
               -H "Authorization: token $GITHUB_TOKEN" \
-               https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/actions/workflows/subsystem-benchmarks.yml/dispatches \
-                -d '{"ref":"refs/heads/master","inputs":{"benchmark-data-dir-path":"'$benchmark_dir'","output-file-path":"'$bencmark_name'"}}'
-        sleep 300
+              https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/actions/workflows/subsystem-benchmarks.yml/dispatches \
+              -d "{\"ref\":\"refs/heads/master\",\"inputs\":{\"benchmark-data-dir-path\":\"$benchmark_dir\",\"output-file-path\":\"$benchmark_name\"}}";
+        sleep 300;
       done
   allow_failure: true
 

.gitlab/pipeline/test.yml

@@ -23,9 +23,8 @@ test-linux-stable:
     - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}"
     # add experimental to features after https://github.com/paritytech/substrate/pull/14502 is merged
     # "upgrade_version_checks_should_work" is currently failing
-    - |
+    - >
       time cargo nextest run \
-        --filter-expr 'not deps(/polkadot-subsystem-bench/)' \
         --workspace \
         --locked \
         --release \
@@ -35,7 +34,7 @@ test-linux-stable:
     # Upload tests results to Elasticsearch
     - echo "Upload test results to Elasticsearch"
     - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
-    - |
+    - >
       curl -v -XPOST --http1.1 \
       -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} \
       https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} \
@@ -70,7 +69,7 @@ test-linux-stable-runtime-benchmarks:
     # but still want to have debug assertions.
     RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
   script:
-    - time cargo nextest run --filter-expr 'not deps(/polkadot-subsystem-bench/)' --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet
+    - time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet
 
 # can be used to run all tests
 # test-linux-stable-all:
@@ -88,7 +87,7 @@ test-linux-stable-runtime-benchmarks:
 #   script:
 #     # Build all but only execute 'runtime' tests.
 #     - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}"
-#     - |
+#     - >
 #       time cargo nextest run \
 #         --workspace \
 #         --locked \
@@ -323,7 +322,24 @@ quick-benchmarks:
     WASM_BUILD_NO_COLOR: 1
     WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
   script:
-    - time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
+    - time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks --quiet -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet
+
+quick-benchmarks-omni:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-C debug-assertions"
+    RUST_BACKTRACE: "full"
+    WASM_BUILD_NO_COLOR: 1
+    WASM_BUILD_RUSTFLAGS: "-C debug-assertions"
+  script:
+    - time cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
+    - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
 
 test-frame-examples-compile-to-wasm:
   # into one job