[8.18] (backport #18036) Split integration tests into more parts (#18038)

* Split integration tests into more parts (#18036)

* Split integration tests into 4 parts

Previously we were splitting in to 3 parts. The integration tests run in about
25 minutes. This commit updates the split to try to bring that time solidly
under 20 minutes in CI.

* Split integration testin into 6 parts

Previously the tests were split into 3. splitting to 4 offered a small
improvement. See how 6 does in buildkite.

(cherry picked from commit 19d9914)

# Conflicts:
#	.buildkite/pull_request_pipeline.yml

* Fix merge conflict

---------

Co-authored-by: Cas Donoghue <[email protected]>

Author: mergify[bot]

.buildkite/pull_request_pipeline.yml

@@ -79,8 +79,8 @@ steps:
       manual:
         allowed: true
 
-  - label: ":lab_coat: Integration Tests / part 1-of-3"
-    key: "integration-tests-part-1-of-3"
+  - label: ":lab_coat: Integration Tests / part 1-of-6"
+    key: "integration-tests-part-1-of-6"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -95,10 +95,10 @@ steps:
       set -euo pipefail
 
       source .buildkite/scripts/common/container-agent.sh
-      ci/integration_tests.sh split 0 3
+      ci/integration_tests.sh split 0 6
 
-  - label: ":lab_coat: Integration Tests / part 2-of-3"
-    key: "integration-tests-part-2-of-3"
+  - label: ":lab_coat: Integration Tests / part 2-of-6"
+    key: "integration-tests-part-2-of-6"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -113,10 +113,10 @@ steps:
       set -euo pipefail
 
       source .buildkite/scripts/common/container-agent.sh
-      ci/integration_tests.sh split 1 3
+      ci/integration_tests.sh split 1 6
 
-  - label: ":lab_coat: Integration Tests / part 3-of-3"
-    key: "integration-tests-part-3-of-3"
+  - label: ":lab_coat: Integration Tests / part 3-of-6"
+    key: "integration-tests-part-3-of-6"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -131,10 +131,121 @@ steps:
       set -euo pipefail
 
       source .buildkite/scripts/common/container-agent.sh
-      ci/integration_tests.sh split 2 3
+      ci/integration_tests.sh split 2 6
 
-  - label: ":lab_coat: IT Persistent Queues / part 1-of-3"
-    key: "integration-tests-qa-part-1-of-3"
+  - label: ":lab_coat: Integration Tests / part 4-of-6"
+    key: "integration-tests-part-4-of-6"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      cpu: "8"
+      memory: "16Gi"
+      ephemeralStorage: "100Gi"
+      # Run as a non-root user
+      imageUID: "1002"
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      ci/integration_tests.sh split 3 6
+
+  - label: ":lab_coat: Integration Tests / part 5-of-6"
+    key: "integration-tests-part-5-of-6"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      cpu: "8"
+      memory: "16Gi"
+      ephemeralStorage: "100Gi"
+      # Run as a non-root user
+      imageUID: "1002"
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      ci/integration_tests.sh split 4 6
+
+  - label: ":lab_coat: Integration Tests / part 6-of-6"
+    key: "integration-tests-part-6-of-6"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      cpu: "8"
+      memory: "16Gi"
+      ephemeralStorage: "100Gi"
+      # Run as a non-root user
+      imageUID: "1002"
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      ci/integration_tests.sh split 5 6
+
+  - label: ":lab_coat: IT Persistent Queues / part 1-of-6"
+    key: "integration-tests-qa-part-1-of-6"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      cpu: "8"
+      memory: "16Gi"
+      ephemeralStorage: "100Gi"
+      # Run as non root (logstash) user. UID is hardcoded in image.
+      imageUID: "1002"
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      export FEATURE_FLAG=persistent_queues
+      ci/integration_tests.sh split 0 6
+
+  - label: ":lab_coat: IT Persistent Queues / part 2-of-6"
+    key: "integration-tests-qa-part-2-of-6"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      cpu: "8"
+      memory: "16Gi"
+      ephemeralStorage: "100Gi"
+      # Run as non root (logstash) user. UID is hardcoded in image.
+      imageUID: "1002"
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      export FEATURE_FLAG=persistent_queues
+      ci/integration_tests.sh split 1 6
+
+  - label: ":lab_coat: IT Persistent Queues / part 3-of-6"
+    key: "integration-tests-qa-part-3-of-6"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      cpu: "8"
+      memory: "16Gi"
+      ephemeralStorage: "100Gi"
+      # Run as non root (logstash) user. UID is hardcoded in image.
+      imageUID: "1002"
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      export FEATURE_FLAG=persistent_queues
+      ci/integration_tests.sh split 2 6
+
+  - label: ":lab_coat: IT Persistent Queues / part 4-of-6"
+    key: "integration-tests-qa-part-4-of-6"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -150,10 +261,10 @@ steps:
 
       source .buildkite/scripts/common/container-agent.sh
       export FEATURE_FLAG=persistent_queues
-      ci/integration_tests.sh split 0 3
+      ci/integration_tests.sh split 3 6
 
-  - label: ":lab_coat: IT Persistent Queues / part 2-of-3"
-    key: "integration-tests-qa-part-2-of-3"
+  - label: ":lab_coat: IT Persistent Queues / part 5-of-6"
+    key: "integration-tests-qa-part-5-of-6"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -169,10 +280,10 @@ steps:
 
       source .buildkite/scripts/common/container-agent.sh
       export FEATURE_FLAG=persistent_queues
-      ci/integration_tests.sh split 1 3
+      ci/integration_tests.sh split 4 6
 
-  - label: ":lab_coat: IT Persistent Queues / part 3-of-3"
-    key: "integration-tests-qa-part-3-of-3"
+  - label: ":lab_coat: IT Persistent Queues / part 6-of-6"
+    key: "integration-tests-qa-part-6-of-6"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -188,7 +299,7 @@ steps:
 
       source .buildkite/scripts/common/container-agent.sh
       export FEATURE_FLAG=persistent_queues
-      ci/integration_tests.sh split 2 3
+      ci/integration_tests.sh split 5 6
 
   - label: ":lab_coat: x-pack unit tests"
     key: "x-pack-unit-tests"