[CI] Refactors Buildkite to move main testing to YAML runner

Author: picandocodigo

.buildkite/pipeline.yml

@@ -1,5 +1,5 @@
 steps:
-  - label: ":ruby: {{ matrix.ruby_source}}:{{ matrix.ruby }} :phone: Transport: {{ matrix.transport }} :elasticsearch: Suite: {{ matrix.suite }}"
+  - label: ":elasticsearch: Elasticsearch :ruby: {{ matrix.ruby_source}}:{{ matrix.ruby }} :phone: Transport {{ matrix.transport }} {{ matrix.suite }}"
     agents:
       provider: "gcp"
     env:
@@ -14,63 +14,41 @@ steps:
         suite:
           - "free"
           - "platinum"
+        ruby:
+          - "3.3"
+        transport:
+          - "8.3"
+        ruby_source:
+          - "ruby"
+    command: ./.buildkite/run-tests.sh
+    # I'm publishing test results to HTML and JUnit in this directory and this directive makes them
+    # available in the Artifacts tab of a build in Buildkite.
+    artifact_paths: "elasticsearch-api/tmp/*"
+  - label: ":yaml: YAML test suite :ruby: {{ matrix.ruby_source}}:{{ matrix.ruby }} :phone: Transport {{ matrix.transport }}"
+    agents:
+      provider: "gcp"
+    matrix:
+      setup:
         ruby:
           - "3.3"
           - "3.2"
           - "3.1"
           - "3.0"
+        ruby_source:
+          - "ruby"
         transport:
           - "8.3"
-        ruby_source:
-          - 'ruby'
-      # Only run platinum once for the latest Ruby. You can use lots of combinations, check the
-      # documentation on https://buildkite.com/docs/pipelines/build-matrix for more information.
       adjustments:
         - with:
-            suite: "platinum"
-            ruby: "3.2"
-          skip: true
-        - with:
-            suite: "platinum"
-            ruby: "3.1"
-          skip: true
-        - with:
-            suite: "platinum"
-            ruby: "3.0"
-          skip: true
-        - with:
-            suite: "free"
-            ruby: "3.3"
-            ruby_source: "ruby"
-            transport: "8.1"
-        - with:
-            suite: "free"
-            ruby: "3.3"
-            ruby_source: "ruby"
-            transport: "8.2"
-        - with:
-            suite: "free"
-            ruby: "3.3"
-            ruby_source: "ruby"
-            transport: "main"
-        - with:
-            suite: "free"
             ruby: "9.4"
             ruby_source: "jruby"
             transport: "8.3"
-    command: ./.buildkite/run-tests.sh
-    # I'm publishing test results to HTML and JUnit in this directory and this directive makes them
-    # available in the Artifacts tab of a build in Buildkite.
-    artifact_paths: "elasticsearch-api/tmp/*"
-  - label: ":ruby: :yaml: Test suite"
-    agents:
-      provider: "gcp"
     env:
-      RUBY_VERSION: "3.3"
+      RUBY_VERSION: "{{ matrix.ruby }}"
       STACK_VERSION: 9.0.0-SNAPSHOT
       ES_YAML_TESTS_BRANCH: main
-      TRANSPORT_VERSION: "8.3"
-      RUBY_SOURCE: "ruby"
+      TRANSPORT_VERSION: "{{ matrix.transport }}"
+      RUBY_SOURCE: "{{ matrix.ruby_source }}"
       TEST_SUITE: "platinum"
     command: ./.buildkite/run-yaml-tests.sh
   - wait: ~