Merge branch 'main' into sagemaker-openai-completion

Author: prwhelan

.buildkite/pipelines/periodic.template.yml

@@ -78,8 +78,8 @@ steps:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
   - group: java-matrix
     steps:
-      - label: "{{matrix.ES_RUNTIME_JAVA}} / entitlements={{matrix.ENTITLEMENTS_ENABLED}} / {{matrix.GRADLE_TASK}} / java-matrix"
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline="-Des.entitlements.enabled=$$ENTITLEMENTS_ENABLED" $$GRADLE_TASK
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $$GRADLE_TASK
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -94,9 +94,6 @@ steps:
               - checkPart4
               - checkPart5
               - checkRestCompat
-            ENTITLEMENTS_ENABLED:
-              - "true"
-              - "false"
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -105,7 +102,6 @@ steps:
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
-          ENTITLEMENTS_ENABLED: "{{matrix.ENTITLEMENTS_ENABLED}}"
       - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-matrix-bwc"
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -497,8 +497,8 @@ steps:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
   - group: java-matrix
     steps:
-      - label: "{{matrix.ES_RUNTIME_JAVA}} / entitlements={{matrix.ENTITLEMENTS_ENABLED}} / {{matrix.GRADLE_TASK}} / java-matrix"
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline="-Des.entitlements.enabled=$$ENTITLEMENTS_ENABLED" $$GRADLE_TASK
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $$GRADLE_TASK
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -513,9 +513,6 @@ steps:
               - checkPart4
               - checkPart5
               - checkRestCompat
-            ENTITLEMENTS_ENABLED:
-              - "true"
-              - "false"
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -524,7 +521,6 @@ steps:
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
-          ENTITLEMENTS_ENABLED: "{{matrix.ENTITLEMENTS_ENABLED}}"
       - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-matrix-bwc"
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
         timeout_in_minutes: 300

.buildkite/pipelines/pull-request/part-1-entitlements.yml

@@ -7,6 +7,7 @@ set -euo pipefail
 # The second/lowercase export is what the tests expect/require
 
 if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
+  # These credentials expire periodically and must be manually renewed - the process is in the onboarding/process docs.
   json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_test_creds)
 
   AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)
@@ -19,6 +20,7 @@ if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
 fi
 
 if [[ "${USE_3RD_PARTY_AZURE_SAS_CREDENTIALS:-}" == "true" ]]; then
+  # These credentials expire periodically and must be manually renewed - the process is in the onboarding/process docs.
   json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_sas_test_creds)
 
   AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)

.buildkite/pipelines/pull-request/part-2-entitlements.yml

@@ -97,6 +97,15 @@ From the `elastic-start-local` folder, check the connection to Elasticsearch usi
 source .env
 curl $ES_LOCAL_URL -H "Authorization: ApiKey ${ES_LOCAL_API_KEY}"
 ----
+
+To use the password for the `elastic` user, set and export the `ES_LOCAL_PASSWORD` environment variable. For example:
+
+[source,sh]
+----
+source .env
+export ES_LOCAL_PASSWORD
+----
+
 // NOTCONSOLE
 
 === Send requests to Elasticsearch
@@ -112,22 +121,23 @@ Here's an example curl command to create a new Elasticsearch index, using basic
 
 [source,sh]
 ----
-curl -u elastic:$ELASTIC_PASSWORD \
+curl -u elastic:$ES_LOCAL_PASSWORD \
   -X PUT \
   http://localhost:9200/my-new-index \
   -H 'Content-Type: application/json'
 ----
+
 // NOTCONSOLE
 
 ==== Using a language client
 
-To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable.
+To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password stored in the `ES_LOCAL_PASSWORD` environment variable.
 
 You'll use the following connection details:
 
 * **Elasticsearch endpoint**: `http://localhost:9200`
 * **Username**: `elastic`
-* **Password**: `$ELASTIC_PASSWORD` (Value you set in the environment variable)
+* **Password**: `$ES_LOCAL_PASSWORD` (Value you set in the environment variable)
 
 For example, to connect with the Python `elasticsearch` client:
 
@@ -137,7 +147,7 @@ import os
 from elasticsearch import Elasticsearch
 
 username = 'elastic'
-password = os.getenv('ELASTIC_PASSWORD') # Value you set in the environment variable
+password = os.getenv('ES_LOCAL_PASSWORD') # Value you set in the environment variable
 
 client = Elasticsearch(
     "http://localhost:9200",

.buildkite/pipelines/pull-request/part-3-entitlements.yml

@@ -46,14 +46,8 @@ run it using Gradle:
 ==== Launching and debugging from an IDE
 
 If you want to run and debug Elasticsearch from your IDE, the `./gradlew run` task
-supports a remote debugging option. Run the following from your terminal:
-
----------------------------------------------------------------------------
-./gradlew run --debug-jvm
----------------------------------------------------------------------------
-
-Next start the "Debug Elasticsearch" run configuration in IntelliJ. This will enable the IDE to connect to the process and allow debug functionality.
-
+supports a remote debugging option. Start the "Debug Elasticsearch" run configuration in IntelliJ. This will enable the
+IDE to connect to the process and allow debug functionality.
 
 As such the IDE needs to be instructed to listen for connections on the debug port.
 Since we might run multiple JVMs as part of configuring and starting the cluster it's
@@ -64,6 +58,12 @@ NOTE: If you have imported the project into IntelliJ according to the instructio
 link:/CONTRIBUTING.md#importing-the-project-into-intellij-idea[CONTRIBUTING.md] then a debug run configuration
 named "Debug Elasticsearch" will be created for you and configured appropriately.
 
+Next run the following from your terminal:
+
+---------------------------------------------------------------------------
+./gradlew run --debug-jvm
+---------------------------------------------------------------------------
+
 ===== Debugging the CLI launcher
 
 The gradle task does not start the Elasticsearch server process directly; like in the Elasticsearch distribution,