Add PR performance gate (#8970)

* feat(ci): PR performance gate

* chore(ci): Review comments

* feat(ci): Explicitly rely on job artifacts, drop dacapo

* fix(ci): revert removal of ARTIFACTSDIR for bp-runner.fail-on-regression.yml path

* chore(ci): replace ARTIFACTSDIR by CI_PROJECT_DIR for bp-runner.fail-on-regression.yml path

* chore(ci): Move threshold to 10%

* chore(ci): Allow failure on PR gate to exercise it

* chore(ci): Disable failure but increase regression threshold (to be refined)

* chore(ci): Choose 10% as threshold.

20% was chosen due to variability of the results on unrelated changes.

After discussion in lp-sync, 20% won't get us to move forward and entice to fix the problems. So instead choose a closer threshold to 10%.

Author: bric3

.gitlab/benchmarks.yml

@@ -68,6 +68,36 @@ benchmarks-post-results:
     - job: benchmarks-dacapo
       artifacts: true
 
+check-big-regressions:
+  extends: .benchmarks
+  needs:
+    - job: benchmarks-startup
+      artifacts: true
+    - job: benchmarks-load
+      artifacts: true
+  when: on_success
+  tags: ["arch:amd64"]
+  # ARTIFACTS_DIR /go/src/github.com/DataDog/apm-reliability/dd-trace-java/reports/
+  # need to convert them
+  script:
+    - !reference [ .benchmarks, script ]
+    - | 
+      for benchmarkType in startup load; do
+          find "$ARTIFACTS_DIR/$benchmarkType" -name "benchmark-baseline.json" -o -name "benchmark-candidate.json" | while read file; do
+            relpath="${file#$ARTIFACTS_DIR/$benchmarkType/}"
+            prefix="${relpath%/benchmark-*}" # Remove the trailing /benchmark-(baseline|candidate).json
+            prefix="${prefix#./}" # Remove any leading ./
+            prefix="${prefix//\//-}" # Replace / with -
+            case "$file" in
+              *benchmark-baseline.json) type="baseline" ;;
+              *benchmark-candidate.json) type="candidate" ;;
+            esac
+            echo "Moving $file to $ARTIFACTS_DIR/${type}-${prefix}.converted.json"
+            cp "$file" "$ARTIFACTS_DIR/${type}-${prefix}.converted.json"
+          done
+      done
+    - bp-runner $CI_PROJECT_DIR/.gitlab/benchmarks/bp-runner.fail-on-regression.yml --debug
+
 .dsm-kafka-benchmarks:
   stage: benchmarks
   rules:

.gitlab/benchmarks/bp-runner.fail-on-regression.yml

@@ -0,0 +1,7 @@
+experiments:
+  - name: Run regression check
+    steps:
+      - name: Regression Check
+        run: fail_on_regression
+        # Applies on all scenarios
+        regression_threshold: 10.0 # percents