[ci] Remove runtime-benchmarks from tests (#1335)

* [ci] Remove runtime-benchmarks from tests

* Update .gitlab/pipeline/test.yml

Co-authored-by: Oliver Tale-Yazdi <[email protected]>

* remove ing

* move benchmark tests to additional tests

* rm -q option

* try release profile

* use testnet profile

* move to a separate job

* rm dup

---------

Co-authored-by: Oliver Tale-Yazdi <[email protected]>

Author: 2 people

.gitlab/pipeline/test.yml

@@ -34,7 +34,7 @@ test-linux-stable:
         --locked \
         --release \
         --no-fail-fast \
-        --features runtime-benchmarks,try-runtime,experimental \
+        --features try-runtime,experimental \
         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
     # run runtime-api tests with `enable-staging-api` feature on the 1st node
     - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi
@@ -92,7 +92,22 @@ test-linux-stable-additional-tests:
         --locked \
         --release \
         --features runtime-benchmarks,try-runtime
-  allow_failure: true
+
+# https://github.com/paritytech/ci_cd/issues/864
+test-linux-stable-runtime-benchmarks:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+    - .pipeline-stopper-artifacts
+  variables:
+    RUST_TOOLCHAIN: stable
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+  script:
+    - time cargo nextest run --features runtime-benchmarks benchmark --locked --cargo-profile testnet
 
 # these ones can be really slow so it's better to run them separately
 test-linux-stable-slow: