Merge RUN_E2E_TESTS and RUN_KITCHEN_TESTS variables (#23613)

* Merge RUN_E2E_TESTS and RUN_KITCHEN_TESTS variables

* Rename more rules

* Rename test var

* Fix gitlab-ci file

* Fix if_not_e2e

* Rename if not e2e

* Rename if not e2e

* Fix yaml aliases

* Update pipeline run task

* Rename alias run_e2e_tests

* fixI(CI): Replace test with false by off

* fixI(CI): Condition to not trigger windows PR test

* fix(invoke): change RUN_E2E_TESTS values

* fix(CI): Fix variable name

* Update .gitlab-ci.yml

Co-authored-by: Nicolas Schweitzer <[email protected]>

* doc(CI): Explain behavior of auto

---------

Co-authored-by: Nicolas Schweitzer <[email protected]>

Author: KevinFairise2

.gitlab-ci.yml

@@ -209,7 +209,7 @@ variables:
   CLANG_LLVM_VER: 12.0.1
   KERNEL_MATRIX_TESTING_X86_AMI_ID: "ami-0c54d42f8f4180b0c"
   KERNEL_MATRIX_TESTING_ARM_AMI_ID: "ami-021f04c00ecfa8590"
-  RUN_E2E_TESTS: "manual" # Should be "auto", "manual", "off" it will change the trigger condition for new-e2e tests on branch != main
+  RUN_E2E_TESTS: "auto" # Should be "off", "auto" or "on" it will change the trigger condition for new-e2e tests on branch != main
   # skip known flaky tests by default
   GO_TEST_SKIP_FLAKE: "true"
 
@@ -282,31 +282,29 @@ variables:
 .if_not_run_all_builds: &if_not_run_all_builds
   if: $CI_COMMIT_BRANCH != "main" && $DEPLOY_AGENT != "true" && $RUN_ALL_BUILDS != "true"
 
-# Rule to trigger test kitchen setup, run, and cleanup.
+# Rule to trigger test setup, run, and cleanup.
 # By default:
-# - on main and deploy pipelines, kitchen tests are run
-# - on branch pipelines, kitchen tests are not run
-# RUN_KITCHEN_TESTS can be set to true to force kitchen tests to be run on a branch pipeline.
-# RUN_KITCHEN_TESTS can be set to false to force kitchen tests to not run on main/deploy pipelines.
-.if_kitchen: &if_kitchen
-  if: ($CI_COMMIT_BRANCH == "main"  || $DEPLOY_AGENT == "true" || $RUN_KITCHEN_TESTS == "true") && $RUN_KITCHEN_TESTS != "false"
-
-# Rules to trigger default kitchen tests.
-# Some of the kitchen tests are run on all pipelines by default. They can only be disabled
-# by setting RUN_KITCHEN_TESTS to false.
-.if_default_kitchen: &if_default_kitchen
-  if: $RUN_KITCHEN_TESTS != "false"
+# - on main and deploy pipelines, installer tests are run
+# - on branch pipelines, installer tests are run on a subset of the OSes we test
+# RUN_E2E_TESTS can be set to on to force all the installer tests to be run on a branch pipeline.
+# RUN_E2E_TESTS can be set to false to force installer tests to not run on main/deploy pipelines.
+.if_installer_tests: &if_installer_tests
+  if: ($CI_COMMIT_BRANCH == "main"  || $DEPLOY_AGENT == "true" || $RUN_E2E_TESTS == "on") && $RUN_E2E_TESTS != "off"
 
 .if_testing_cleanup: &if_testing_cleanup
   if: $TESTING_CLEANUP == "true"
 
-.if_not_e2e: &if_not_e2e
-  if: $RUN_KITCHEN_TESTS == "false"
+.if_run_all_e2e_tests: &if_run_all_e2e_tests
+  if: $RUN_E2E_TESTS == "on"
 
-.if_auto_e2e: &if_auto_e2e
+# When RUN_E2E_TESTS is set to "auto". We do not enforce a behavior for the tests.
+# The behavior of each test will be defined by its rules.
+# For example for new-e2e tests created by each team, here is an example of such rules: https://github.com/DataDog/datadog-agent/blob/ba7079d92077ab5898378594dcafb9cd88a77e57/.gitlab-ci.yml#L1160-L1167
+# For the installer tests when RUN_E2E_TESTS is set to "auto", we run a subset of tests on branch pipelines and all the tests on main.
+.if_auto_e2e_tests: &if_auto_e2e_tests
   if: $RUN_E2E_TESTS == "auto"
 
-.if_disable_e2e: &if_disable_e2e
+.if_disable_e2e_tests: &if_disable_e2e_tests
   if: $RUN_E2E_TESTS == "off"
 
 .if_deploy_on_beta_repo_branch: &if_deploy_on_beta_repo_branch
@@ -797,7 +795,7 @@ workflow:
   - when: on_success
 
 .except_no_tests_no_deploy:
-  - if: $RUN_KITCHEN_TESTS == "false" && $DEPLOY_AGENT == "false" && $RUN_E2E_TESTS == "off"
+  - if: $DEPLOY_AGENT == "false" && $RUN_E2E_TESTS == "off"
     when: never
 
 .on_a6_except_deploy:
@@ -864,45 +862,39 @@ workflow:
 .on_kitchen_tests_a6:
   - <<: *if_not_version_6
     when: never
-  - <<: *if_kitchen
+  - <<: *if_installer_tests
 
 .on_kitchen_tests_a6_always:
   - <<: *if_not_version_6
     when: never
-  - <<: *if_kitchen
+  - <<: *if_installer_tests
     when: always
 
 .on_all_kitchen_builds_a6:
   - <<: *if_not_version_6
     when: never
   - <<: *if_not_run_all_builds
     when: never
-  - <<: *if_kitchen
+  - <<: *if_installer_tests
 
 .on_kitchen_tests_a7:
   - <<: *if_not_version_7
     when: never
-  - <<: *if_kitchen
-
-.on_kitchen_tests_a7_always:
-  - <<: *if_not_version_7
-    when: never
-  - <<: *if_kitchen
-    when: always
+  - <<: *if_installer_tests
 
 .on_all_kitchen_builds_a7:
   - <<: *if_not_version_7
     when: never
   - <<: *if_not_run_all_builds
     when: never
-  - <<: *if_kitchen
+  - <<: *if_installer_tests
 
 .on_all_new-e2e_tests_a7:
   - <<: *if_not_version_7
     when: never
   - <<: *if_not_run_all_builds
     when: never
-  - <<: *if_kitchen
+  - <<: *if_installer_tests
 
 # Default kitchen tests are also run on dev branches
 # In that case, the target OS versions is a subset of the
@@ -912,8 +904,8 @@ workflow:
     when: never
   - <<: *if_not_version_7
     when: never
-  - <<: *if_kitchen
-  - <<: *if_default_kitchen
+  - <<: *if_installer_tests
+  - <<: *if_auto_e2e_tests
     variables:
       KITCHEN_OSVERS: $DEFAULT_KITCHEN_OSVERS
 
@@ -922,8 +914,10 @@ workflow:
     when: never
   - <<: *if_not_version_7
     when: never
-  - <<: *if_kitchen
-  - <<: *if_default_kitchen
+  - <<: *if_disable_e2e_tests
+    when: never
+  - <<: *if_installer_tests
+  - <<: *if_auto_e2e_tests
     variables:
       E2E_OSVERS: $E2E_BRANCH_OSVERS
 
@@ -932,9 +926,9 @@ workflow:
     when: never
   - <<: *if_not_version_7
     when: never
-  - <<: *if_kitchen
+  - <<: *if_installer_tests
     when: always
-  - <<: *if_default_kitchen
+  - <<: *if_auto_e2e_tests
     when: always
     variables:
       KITCHEN_OSVERS: $DEFAULT_KITCHEN_OSVERS
@@ -986,7 +980,7 @@ workflow:
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
 .on_windows_installer_changes_or_manual:
-  - <<: *if_not_e2e
+  - <<: *if_disable_e2e_tests
     when: never
   - <<: *if_main_branch
   - <<: *if_mergequeue
@@ -1049,11 +1043,11 @@ workflow:
 
 
 .on_e2e_main_release_or_rc: # This rule is used as a base for all new-e2e rules
-  - <<: *if_disable_e2e
+  - <<: *if_disable_e2e_tests
     when: never
   - <<: *if_mergequeue
     when: never
-  - <<: *if_auto_e2e
+  - <<: *if_run_all_e2e_tests
     when: on_success
   - <<: *if_main_branch
     when: on_success
@@ -1068,11 +1062,11 @@ workflow:
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
 .always_on_container_or_e2e_changes_or_manual:
-  - <<: *if_disable_e2e
+  - <<: *if_disable_e2e_tests
     when: never
   - <<: *if_mergequeue
     when: never
-  - <<: *if_auto_e2e
+  - <<: *if_run_all_e2e_tests
     when: always
   - <<: *if_main_branch
     when: always
@@ -1252,7 +1246,7 @@ workflow:
     when: never
   - <<: *if_not_main_branch
     when: never
-  - <<: *if_kitchen
+  - <<: *if_installer_tests
     when: manual
     allow_failure: true
 
@@ -1271,7 +1265,7 @@ workflow:
     allow_failure: true
 
 .on_updater_or_e2e_changes_or_manual:
-  - <<: *if_disable_e2e
+  - <<: *if_disable_e2e_tests
     when: never
   - <<: *if_mergequeue
     when: never
@@ -1310,7 +1304,7 @@ workflow:
     allow_failure: true
 
 .on_main_or_rc_and_no_skip_e2e:
-  - <<: *if_not_e2e
+  - <<: *if_disable_e2e_tests
     when: never
   - <<: *if_release_branch
     when: on_success
@@ -1323,11 +1317,6 @@ workflow:
   - <<: *if_mergequeue
     when: never
 
-.if_run_e2e_tests:
-  - <<: *if_disable_e2e
-    when: never
-  - <<: *if_auto_e2e
-
 .on_packaging_change:
   - !reference [.except_mergequeue] # The prerequisites are not run in the mergequeue pipeline so we need to skip this rule
   - changes:

tasks/libs/pipeline_tools.py

@@ -87,7 +87,6 @@ def trigger_agent_pipeline(
     branch="nightly",
     deploy=False,
     all_builds=False,
-    kitchen_tests=False,
     e2e_tests=False,
     rc_build=False,
     rc_k8s_deployments=False,
@@ -111,17 +110,12 @@ def trigger_agent_pipeline(
     if all_builds:
         args["RUN_ALL_BUILDS"] = "true"
 
-    # Kitchen tests can be selectively enabled, or disabled on pipelines where they're
-    # enabled by default (default branch and deploy pipelines).
-    if kitchen_tests:
-        args["RUN_KITCHEN_TESTS"] = "true"
-    else:
-        args["RUN_KITCHEN_TESTS"] = "false"
-
     # End to end tests can be selectively enabled, or disabled on pipelines where they're
     # enabled by default (default branch and deploy pipelines).
     if e2e_tests:
-        args["RUN_E2E_TESTS"] = "true"
+        args["RUN_E2E_TESTS"] = "on"
+    else:
+        args["RUN_E2E_TESTS"] = "off"
 
     if release_version_6 is not None:
         args["RELEASE_VERSION_6"] = release_version_6

tasks/pipeline.py

@@ -221,8 +221,7 @@ def run(
     repo_branch="dev",
     deploy=False,
     all_builds=True,
-    kitchen_tests=True,
-    e2e_tests=False,
+    e2e_tests=True,
     rc_build=False,
     rc_k8s_deployments=False,
 ):
@@ -302,14 +301,14 @@ def run(
                 )
             )
             all_builds = True
-        if not kitchen_tests:
+        if not e2e_tests:
             print(
                 color_message(
-                    "WARNING: ignoring --no-kitchen-tests option, RUN_KITCHEN_TESTS is automatically set to true on deploy pipelines",
+                    "WARNING: ignoring --no-e2e-tests option, RUN_E2E_TESTS is automatically set to true on deploy pipelines",
                     "orange",
                 )
             )
-            kitchen_tests = True
+            e2e_tests = True
 
     pipelines = get_running_pipelines_on_same_ref(gitlab, git_ref)
 
@@ -332,7 +331,6 @@ def run(
             repo_branch,
             deploy=deploy,
             all_builds=all_builds,
-            kitchen_tests=kitchen_tests,
             e2e_tests=e2e_tests,
             rc_build=rc_build,
             rc_k8s_deployments=rc_k8s_deployments,