ci: add manual break glass options for tests and benchmarks (#13222) (#13349)

Backport of #13222 to `2.21`

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: brettlangdon

.gitlab-ci.yml

@@ -10,6 +10,10 @@ stages:
 
 variables:
   REPO_LANG: python # "python" is used everywhere rather than "py"
+  REPO_NOTIFICATION_CHANNEL: "#apm-python-release"
+  RELEASE_ALLOW_TEST_FAILURES: false
+  RELEASE_ALLOW_BENCHMARK_FAILURES: false
+  DD_DISABLE_VPA: true  # disable only on 2.21/older release branches
   # CI_DEBUG_SERVICES: "true"
 
 default:
@@ -38,6 +42,11 @@ tests-gen:
 run-tests-trigger:
  stage: tests-trigger
  needs: [ tests-gen ]
+ # Allow the child job to fail if explicitly asked
+ rules:
+   - if: $RELEASE_ALLOW_TEST_FAILURES == "true"
+     allow_failure: true
+   - allow_failure: false
  trigger:
    include:
      - artifact: .gitlab/tests-gen.yml
@@ -108,7 +117,14 @@ publish-lib-init-pinned-tags:
 configure_system_tests:
   variables:
     SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,docker-ssi,lib-injection"
-    
+
+system_tests:
+  # Allow the child job to fail if explicitly asked
+  rules:
+    - if: $RELEASE_ALLOW_TEST_FAILURES == "true"
+      allow_failure: true
+    - allow_failure: false
+
 deploy_to_reliability_env:
   needs: []
 
@@ -132,12 +148,20 @@ deploy_to_di_backend:manual:
 check_new_flaky_tests:
   stage: quality-gate
   extends: .testrunner
+  # Allow the child job to fail if explicitly asked
+  rules:
+    - if: $RELEASE_ALLOW_TEST_FAILURES == "true"
+      allow_failure: true
+    - if: '$CI_COMMIT_BRANCH =~ /^main$/'
+      when: never
+    - if: '$CI_COMMIT_BRANCH =~ /^[0-9].[0-9]*/'
+      when: never
+    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch.*/'
+      when: never
+    - allow_failure: false
+      when: always
   script:
     - export DD_SITE=datadoghq.com
     - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.${CI_PROJECT_NAME}.dd-api-key-qualitygate --with-decryption --query "Parameter.Value" --out text)
     - export DD_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.${CI_PROJECT_NAME}.dd-app-key-qualitygate --with-decryption --query "Parameter.Value" --out text)
     - datadog-ci gate evaluate
-  except:
-    - main
-    - '[0-9].[0-9]*'
-    - 'mq-working-branch**'