Refactored buildkite tests using run-elasticsearch

Author: ezimuel

.buildkite/Dockerfile

@@ -39,4 +39,4 @@ RUN composer install --no-progress > /dev/null
 
 COPY . .
 
-CMD ["bash", ".buildkite/yaml-tests.sh"]
+CMD ["bash", ".buildkite/yaml-tests.sh"]

.buildkite/pipeline.yml

@@ -8,10 +8,11 @@ steps:
       PHP_VERSION: "{{ matrix.php }}"
       TEST_SUITE: "{{ matrix.suite }}"
       STACK_VERSION: 9.0.0-SNAPSHOT
+      BRANCH_CLIENT_TESTS: 9.0
     matrix:
       setup:
         suite:
-          - "stack"
+          - "platinum"
         php:
           - "8.4-cli"
           - "8.3-cli"

.buildkite/run-repository.sh

@@ -5,7 +5,6 @@
 # TEST_SUITE -- which test suite to run: oss or xpack
 # ELASTICSEARCH_URL -- The url at which elasticsearch is reachable, a default is composed based on STACK_VERSION and TEST_SUITE
 # PHP_VERSION -- PHP version (defined in test-matrix.yml, a default is hardcoded here)
-# ES_LOCAL_PASSWORD -- The Elaticsarch password for the elastic user. Used for testing.
 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
@@ -19,11 +18,7 @@ echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m CONTAINER ${elasticsearch_container}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m PHP_VERSION ${PHP_VERSION}\033[0m"
-
-echo -e "\033[1m>>>>> Running Elasticsearch >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
-
-curl -fsSL https://elastic.co/start-local | sh -s -- -v ${STACK_VERSION} -esonly
-source elastic-start-local/.env
+echo -e "\033[34;1mINFO:\033[0m BRANCH_CLIENT_TESTS ${BRANCH_CLIENT_TESTS}\033[0m"
 
 echo -e "\033[1m>>>>> Build docker container >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
@@ -46,7 +41,8 @@ docker run \
   --env TEST_SUITE=${TEST_SUITE} \
   --env PHP_VERSION=${PHP_VERSION} \
   --env ELASTICSEARCH_URL=${ELASTICSEARCH_URL} \
+  --env BRANCH_CLIENT_TESTS=${BRANCH_CLIENT_TESTS} \
   --ulimit nofile=65535:65535 \
   --name elasticsearch-php \
   --rm \
-  elastic/elasticsearch-php
+  elastic/elasticsearch-php

.buildkite/run-tests

@@ -7,5 +7,17 @@ script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
 
+echo "--- :elasticsearch: Starting Elasticsearch"
+DETACH=true bash $script_path/run-elasticsearch.sh
+
+if [[ -n "$RUNSCRIPTS" ]]; then
+  for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do
+    echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+    CONTAINER_NAME=${RUNSCRIPT} \
+      DETACH=true \
+      bash $script_path/run-${RUNSCRIPT}.sh
+  done
+fi
+
 echo "+++ :php: Run Client"
-bash "$script_path/run-repository.sh"
+bash $script_path/run-repository.sh

.buildkite/yaml-tests.sh

@@ -1,10 +1,16 @@
 #!/usr/bin/env bash
 
 # Clone the elasticsearch-clients-tests repository
-git clone https://github.com/elastic/elasticsearch-clients-tests.git tests/elasticsearch-clients-tests
+git clone -b ${BRANCH_CLIENT_TESTS} https://github.com/elastic/elasticsearch-clients-tests.git tests/elasticsearch-clients-tests
 
 # Build the YAML tests
-php tests/build_es_tests.php tests/elasticsearch-clients-tests/tests ${TEST_SUITE} tests/Yaml
+php tests/build_es_tests.php tests/elasticsearch-clients-tests/tests stack tests/Yaml
 
 # Run YAML tests
-ELASTICSEARCH_URL="http://elastic:${ES_LOCAL_PASSWORD}@localhost:9200" vendor/bin/phpunit -c "phpunit-yaml-${TEST_SUITE}-tests.xml"
+vendor/bin/phpunit -c "phpunit-yaml-stack-tests.xml"
+
+# Remove Yaml tests
+rm -rf tests/Yaml
+
+# Remove elasticsearch-clients-tests folder
+rm -rf tests/elasticsearch-clients-tests

.github/workflows/integration_test.yml

@@ -47,4 +47,4 @@ jobs:
     - name: Integration tests
       run: |
         source elastic-start-local/.env
-        ELASTICSEARCH_URL="http://elastic:${ES_LOCAL_PASSWORD}@localhost:9200" TEST_SUITE="platinum" composer run-script integration-test
+        ELASTICSEARCH_URL="http://elastic:${ES_LOCAL_PASSWORD}@localhost:9200" TEST_SUITE="free" composer run-script integration-test

.github/workflows/yaml_test.yml

@@ -13,6 +13,7 @@ jobs:
         os: [ubuntu-latest]
         es-version: [9.0.0-SNAPSHOT]
         test-suite: [stack]
+        branch-client-tests: [9.0]
 
     steps:
     - name: Checkout
@@ -47,10 +48,10 @@ jobs:
 
     - name: Build PHPUnit tests
       run: |
-        git clone https://github.com/elastic/elasticsearch-clients-tests.git tests/elasticsearch-clients-tests
+        git clone -b ${{ matrix.branch-client-tests }} https://github.com/elastic/elasticsearch-clients-tests.git tests/elasticsearch-clients-tests
         php tests/build_es_tests.php tests/elasticsearch-clients-tests/tests ${{ matrix.test-suite }} tests/Yaml
 
     - name: YAML tests
       run: |
         source elastic-start-local/.env
-        ELASTICSEARCH_URL="http://elastic:${ES_LOCAL_PASSWORD}@localhost:9200" vendor/bin/phpunit -c "phpunit-yaml-${{ matrix.test-suite }}-tests.xml"
+        ELASTICSEARCH_URL="http://elastic:${ES_LOCAL_PASSWORD}@localhost:9200" TEST_SUITE="free" vendor/bin/phpunit -c "phpunit-yaml-${{ matrix.test-suite }}-tests.xml"

.gitignore

@@ -27,6 +27,7 @@ util/cache/
 util/*.zip
 util/output/
 util/rest-spec
+elastic-start-local/
 
 # Doctum docs generator
 /doctum.phar

src/ClientBuilder.php

@@ -389,10 +389,10 @@ public function build(): Client
         }
 
         /**
-         * Elastic cloud optimized with gzip
+         * Elastic cloud or serverless optimized with gzip
          * @see https://github.com/elastic/elasticsearch-php/issues/1241 omit for Symfony HTTP Client    
          */
-        if ((!empty($this->cloudId) || $this->isCloudOrServerless($this->hosts)) && !$this->isSymfonyHttpClient($transport)) {
+        if ((!empty($this->cloudId) || $this->isCloud($this->hosts) || $this->isServerless($this->hosts)) && !$this->isSymfonyHttpClient($transport)) {
             $transport->setHeader('Accept-Encoding', 'gzip');
         }
 
@@ -404,19 +404,14 @@ public function build(): Client
     }
 
     /**
-     * Check if the hosts contains only one url and if it is from
-     * Elastic Cloud or Serverless
+     * Check if the hosts contains an Elastic Cloud url
      */
-    protected function isCloudOrServerless(array $hosts): bool
+    protected function isCloud(array $hosts): bool
     {
         if (empty($hosts) || count($hosts)>1) {
             return false;
         }
         $url = $hosts[0];
-        // Serverless
-        if (preg_match('/\.elastic\.cloud/i', $url)) {
-            return true;
-        }
         // Elastic Cloud gcp
         if (preg_match('/\.cloud\.es\.io/i', $url)) {
             return true;
@@ -428,6 +423,21 @@ protected function isCloudOrServerless(array $hosts): bool
         return false;
     }
 
+     /**
+     * Check if the hosts contains an Elastic Serverless url 
+     */
+    protected function isServerless(array $hosts): bool
+    {
+        if (empty($hosts) || count($hosts)>1) {
+            return false;
+        }
+        $url = $hosts[0];
+        if (preg_match('/\.elastic\.cloud/i', $url)) {
+            return true;
+        }
+        return false;
+    }
+
     /**
      * Returns true if the transport HTTP client is Symfony
      */

tests/Utility.php

@@ -67,16 +67,26 @@ public static function getHost(): ?string
         if (false !== $url) {
             return $url;
         }
-        return 'http://localhost:9200';
+        if (getenv('TEST_SUITE') === 'free') {
+            return 'http://localhost:9200';
+        }
+        return 'https://localhost:9200';
     }
 
     /**
      * Build a Client based on ENV variables
      */
     public static function getClient(): Client
     {
+        if (getenv('TEST_SUITE') === 'free') {
+            return ClientBuilder::create()
+                ->setHosts([self::getHost()])
+                ->build();
+        }
         return ClientBuilder::create()
             ->setHosts([self::getHost()])
+            ->setBasicAuthentication('elastic', 'changeme')
+            ->setSSLVerification(false)
             ->build();
     }